浅谈Memcached中的压缩存储

蔚落 2022-08-13 00:38 180阅读 0赞

原贴:http://blog.alwaysmylove.net/2008/04/09/compress-storage-in-memcache/

## [ 浅谈Memcached中的压缩存储 ][_Memcached_] ##

[PHPMySQL][] 标签: [compress][], [memcached][], [PHPMySQL][PHPMySQL 1]  
 点击次数: 198  2008年04月09日 15:57:45 星期三

用PHP Pecl memcache的哥们应该都知道，memcache client的add和set方法中第三个参数是表示是否用压缩存储的

bool Memcache::add ( string key, mixed var [, int flag [, int expire]] )
    bool Memcache::set ( string key, mixed var [, int flag [, int expire]] )

flag
    Use MEMCACHE_COMPRESSED to store the item compressed (uses zlib).

还有一个比较重要的函数，是设置要压缩阀值的

bool Memcache::setCompressThreshold ( int threshold [, float min_savings] )
    threshold
    Controls the minimum value length before attempting to compress automatically. 
    
    min_saving
    Specifies the minimum amount of savings to actually store the value compressed. The supplied value must be between 0 and 1. Default value is 0.2 giving a minimum 20% compression savings.

其中，threshold是设置开始压缩的最小长度，大于这个长度的串才会尝试自动压缩；  
min\_saving是使用压缩存储的最小节省百分比。当压缩后长度 < 原长度 \* (1 - min\_saving) 时，才会使用压缩存储  
  
废话太多了，看源代码最快了

1.  intmmc\_pool\_store(mmc\_pool\_t \*pool, constchar \*command, intcommand\_len, constchar \*key, intkey\_len, intflags, intexpire, constchar \*value, intvalue\_lenTSRMLS\_DC)/\* \{ \{ \{ \*/
2.  \{ 
3.      mmc\_t \*mmc;
4.      char \*request;
5.      int request\_len, result = -1;
6.      char \*key\_copy = NULL, \*data = NULL;
7.   
8.      if (key\_len > MMC\_KEY\_MAX\_SIZE)\{ 
9.          key = key\_copy = estrndup(key, MMC\_KEY\_MAX\_SIZE);
10.         key\_len = MMC\_KEY\_MAX\_SIZE;
11.     \}
12.  
13.     /\* autocompress large values \*/
14.     if (pool\->compress\_threshold && value\_len >= pool\->compress\_threshold)\{ 
15.         flags |= MMC\_COMPRESSED;
16.     \}
17.  
18.     if (flags & MMC\_COMPRESSED)\{ 
19.         unsigned longdata\_len;
20.  
21.         if (!mmc\_compress(&data, &data\_len, value, value\_lenTSRMLS\_CC))\{ 
22.             mmc\_server\_seterror(mmc, "Failed to compress data", 0);
23.             return \-1;
24.         \}
25.  
26.         /\* was enough space saved to motivate uncompress processing on get \*/
27.         if (data\_len < value\_len \* (1 \- pool\->min\_compress\_savings))\{ 
28.             value = data;
29.             value\_len = data\_len;
30.         \}
31.         else \{ 
32.             flags &= ~MMC\_COMPRESSED;
33.             efree(data);
34.             data = NULL;
35.         \}
36.     \}
37.  
38.     request = emalloc(
39.         command\_len
40.         + 1                /\* space \*/
41.         + key\_len
42.         + 1                /\* space \*/
43.         + MAX\_LENGTH\_OF\_LONG
44.         + 1             /\* space \*/
45.         + MAX\_LENGTH\_OF\_LONG
46.         + 1             /\* space \*/
47.         + MAX\_LENGTH\_OF\_LONG
48.         + sizeof("/r/n") \- 1
49.         + value\_len
50.         + sizeof("/r/n") \- 1
51.         + 1
52.         );
53.  
54.     request\_len = sprintf(request, "%s %s %d %d %d/r/n", command, key, flags, expire, value\_len);
55.  
56.     memcpy(request \+ request\_len, value, value\_len);
57.     request\_len \+= value\_len;
58.  
59.     memcpy(request \+ request\_len, "/r/n", sizeof("/r/n") \- 1);
60.     request\_len \+= sizeof("/r/n") \- 1;
61.  
62.     request\[request\_len\] = '/0';
63.  
64.     while (result < 0 && (mmc = mmc\_pool\_find(pool, key, key\_lenTSRMLS\_CC)) != NULL)\{ 
65.         if ((result = mmc\_server\_store(mmc, request, request\_lenTSRMLS\_CC)) < 0)\{ 
66.             mmc\_server\_failure(mmc TSRMLS\_CC);
67.         \}
68.     \}
69.  
70.     if (key\_copy != NULL)\{ 
71.         efree(key\_copy);
72.     \}
73.  
74.     if (data != NULL)\{ 
75.         efree(data);
76.     \}
77.  
78.     efree(request);
79.  
80.     return result;
81. \}
82. /\* \}\}\} \*/
83.  
84. static intmmc\_compress(char \*\*result, unsignedlong \*result\_len, constchar \*data, intdata\_lenTSRMLS\_DC)/\* \{ \{ \{ \*/
85. \{ 
86.     int status, level = MEMCACHE\_G(compression\_level);
87.  
88.     \*result\_len = data\_len \+ (data\_len / 1000) \+ 25 \+ 1; /\* some magic from zlib.c \*/
89.     \*result = (char \*) emalloc(\*result\_len);
90.  
91.     if (!\*result)\{ 
92.         return 0;
93.     \}
94.  
95.     if (level >= 0)\{ 
96.         status = compress2((unsigned char \*) \*result, result\_len, (unsignedconstchar \*)data, data\_len, level);
97.     \} else\{ 
98.         status = compress((unsigned char \*) \*result, result\_len, (unsignedconstchar \*)data, data\_len);
99.     \}
100.  
101.     if (status == Z\_OK)\{ 
102.         \*result = erealloc(\*result, \*result\_len \+ 1);
103.         (\*result)\[\*result\_len\] = '/0';
104.         return 1;
105.     \}
106.  
107.     switch (status)\{ 
108.         case Z\_MEM\_ERROR:
109.             php\_error\_docref(NULL TSRMLS\_CC, E\_WARNING, "Not enough memory to perform compression");
110.             break;
111.         case Z\_BUF\_ERROR:
112.             php\_error\_docref(NULL TSRMLS\_CC, E\_WARNING, "Not enough room in the output buffer to perform compression");
113.             break;
114.         case Z\_STREAM\_ERROR:
115.             php\_error\_docref(NULL TSRMLS\_CC, E\_WARNING, "Invalid compression level");
116.             break;
117.         default:
118.             php\_error\_docref(NULL TSRMLS\_CC, E\_WARNING, "Unknown error during compression");
119.             break;
120.     \}
121.  
122.     efree(\*result);
123.     return 0;
124. \}
125. /\* \}\}\}\*/

来，做个小测试

1.  <?php
2.  $memObj = new memcache();
3.  $memObj\->connect("127.0.0.1", 12121);          
4.  $memObj\->set("test1", str\_repeat("a", 13), MEMCACHE\_COMPRESSED);
5.  $memObj\->set("test2", str\_repeat("a", 14), MEMCACHE\_COMPRESSED);
6.  ?>

然后，telnet上服务器看效果

$telnet 127.0.0.1 12121
    Trying 127.0.0.1...
    Connected to 127.0.0.1.
    Escape character is '^]'.
    get test1
    VALUE test1 0 13
    aaaaaaaaaaaaa
    END
    get test2
    VALUE test2 2 11
    某些乱码
    END

可以看到，13个重复的a没有被压缩，因为压缩后的长度太大，没有到达节省20%空间的阀值。  
14个重复的a被压缩了，因为压缩后长度为11， 11 < 14 \* (1 - 0.2)  
其中，VALUE test 2 11含义如下  
VALUE key flag length  
flag为用二进制表示，第1位表示序列化，第2位表示压缩，其他保留

[_Memcached_]: http://blog.alwaysmylove.net/2008/04/09/compress-storage-in-memcache/
[PHPMySQL]: http://blog.alwaysmylove.net/category/php/
[compress]: http://blog.alwaysmylove.net/tag/compress/
[memcached]: http://blog.alwaysmylove.net/tag/memcached/
[PHPMySQL 1]: http://blog.alwaysmylove.net/tag/phpmysql/