Great! Thanks a lot Otto :) On Aug 4, 2014, at 6:57 AM, osch...@apache.org wrote:
> Repository: trafficserver > Updated Branches: > refs/heads/master fab202517 -> 083abd4ff > > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/README > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/README > b/plugins/experimental/ats_speed/gzip/README > new file mode 100644 > index 0000000..2e74681 > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/README > @@ -0,0 +1,4 @@ > +This gzip is not compiled, but only here to be able to diff later > +with the official gzip plugin. It is very slightly modified no > +be able to disable caching of its compressed output through a > +response header > \ No newline at end of file > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.cc > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/configuration.cc > b/plugins/experimental/ats_speed/gzip/configuration.cc > new file mode 100644 > index 0000000..b1c499d > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/configuration.cc > @@ -0,0 +1,264 @@ > +/** @file > + > + Transforms content using gzip or deflate > + > + @section license License > + > + Licensed to the Apache Software Foundation (ASF) under one > + or more contributor license agreements. See the NOTICE file > + distributed with this work for additional information > + regarding copyright ownership. The ASF licenses this file > + to you under the Apache License, Version 2.0 (the > + "License"); you may not use this file except in compliance > + with the License. You may obtain a copy of the License at > + > + http://www.apache.org/licenses/LICENSE-2.0 > + > + Unless required by applicable law or agreed to in writing, software > + distributed under the License is distributed on an "AS IS" BASIS, > + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + See the License for the specific language governing permissions and > + limitations under the License. > + */ > + > +#include "configuration.h" > +#include <fstream> > +#include <algorithm> > +#include <vector> > +#include <fnmatch.h> > + > +namespace Gzip { > + using namespace std; > + > + void ltrim_if(string& s, int (* fp) (int)) { > + for (size_t i = 0; i < s.size();) { > + if (fp(s[i])) { > + s.erase(i,1); > + } else { > + break; > + } > + } > + } > + > + void rtrim_if(string& s, int (* fp) (int)) { > + for (ssize_t i = (ssize_t)s.size() - 1; i >= 0; i--) { > + if (fp(s[i])) { > + s.erase(i,1); > + } else { > + break; > + } > + } > + } > + > + void trim_if(string& s, int (* fp) (int)) { > + ltrim_if(s, fp); > + rtrim_if(s, fp); > + } > + > + vector<string> tokenize(const string &s, int (* fp) (int)) { > + vector<string> r; > + string tmp; > + > + for (size_t i = 0; i < s.size(); i++) { > + if ( fp(s[i]) ) { > + if ( tmp.size() ) { > + r.push_back(tmp); > + tmp = ""; > + } > + } else { > + tmp += s[i]; > + } > + } > + > + if ( tmp.size() ) { > + r.push_back(tmp); > + } > + > + return r; > + } > + > + enum ParserState { > + kParseStart, > + kParseCompressibleContentType, > + kParseRemoveAcceptEncoding, > + kParseEnable, > + kParseCache, > + kParseDisallow, > + }; > + > + void Configuration::AddHostConfiguration(HostConfiguration * hc){ > + host_configurations_.push_back(hc); > + } > + > + void HostConfiguration::add_disallow(const std::string & disallow) { > + disallows_.push_back(disallow); > + } > + > + void HostConfiguration::add_compressible_content_type(const std::string & > content_type) { > + compressible_content_types_.push_back(content_type); > + } > + > + HostConfiguration * Configuration::Find(const char * host, int > host_length) { > + HostConfiguration * host_configuration = host_configurations_[0]; > + > + std::string shost(host, host_length); > + > + for (size_t i = 1; i < host_configurations_.size(); i++ ) { > + if (host_configurations_[i]->host() == shost){ > + host_configuration = host_configurations_[i]; > + break; > + } > + } > + > + return host_configuration; > + } > + > + bool HostConfiguration::IsUrlAllowed(const char * url, int url_len) { > + string surl(url, url_len); > + > + for (size_t i = 0; i < disallows_.size(); i++) { > + if ( fnmatch (disallows_[i].c_str(), surl.c_str(), 0) == 0 ) { > + info("url [%s] disabled for compression, matched on pattern [%s]", > + surl.c_str(), disallows_[i].c_str()); > + return false; > + } > + } > + > + return true; > + } > + > + bool HostConfiguration::ContentTypeIsCompressible(const char * > content_type, int content_type_length) { > + string scontent_type(content_type, content_type_length); > + bool is_match = false; > + > + for (size_t i = 0; i < compressible_content_types_.size(); i++) { > + const char* match_string = compressible_content_types_[i].c_str(); > + bool exclude = match_string[0] == '!'; > + if (exclude) { > + match_string++;//skip '!' > + } > + if ( fnmatch (match_string, scontent_type.c_str(), 0) == 0 ) { > + info("compressible content type [%s], matched on pattern [%s]", > + scontent_type.c_str(), compressible_content_types_[i].c_str()); > + is_match = !exclude; > + } > + } > + > + return is_match; > + } > + > + Configuration * Configuration::Parse(const char * path ) { > + string pathstring(path); > + > + // If we have a path and it's not an absolute path, make it relative to > the > + // configuration directory. > + if (!pathstring.empty() && pathstring[0] != '/') { > + pathstring.assign(TSConfigDirGet()); > + pathstring.append("/"); > + pathstring.append(path); > + } > + > + trim_if(pathstring, isspace); > + > + Configuration * c = new Configuration(); > + HostConfiguration * current_host_configuration = new > HostConfiguration(""); > + c->AddHostConfiguration(current_host_configuration); > + current_host_configuration->add_compressible_content_type("text/*"); > + > current_host_configuration->add_compressible_content_type("application/xml*"); > + > current_host_configuration->add_compressible_content_type("*javascript*"); > + > current_host_configuration->add_compressible_content_type("image/svg+xml"); > + > + > + if (pathstring.empty()) { > + return c; > + } > + > + path = pathstring.c_str(); > + info("Parsing file \"%s\"", path); > + std::ifstream f; > + > + size_t lineno = 0; > + > + f.open(path, std::ios::in); > + > + if (!f.is_open()) { > + warning("could not open file [%s], skip",path); > + return c; > + } > + > + enum ParserState state = kParseStart; > + > + while (!f.eof()) { > + std::string line; > + getline(f, line); > + ++lineno; > + > + trim_if(line, isspace); > + if (line.size() == 0) { > + continue; > + } > + > + vector<string> v = tokenize( line, isspace ); > + > + for(size_t i = 0; i < v.size(); i++ ) { > + string token = v[i]; > + trim_if(token, isspace); > + > + //should not happen > + if (!token.size()) continue; > + > + //once a comment is encountered, we are done processing the line > + if (token[0] == '#') break; > + > + switch(state) { > + case kParseStart: > + if ( (token[0] == '[') && (token[token.size()-1] == ']')){ > + std::string current_host = token.substr(1,token.size()-2); > + current_host_configuration = new HostConfiguration(current_host); > + c->AddHostConfiguration(current_host_configuration); > + } else if (token == "compressible-content-type" ) { > + state = kParseCompressibleContentType; > + } else if (token == "remove-accept-encoding" ) { > + state = kParseRemoveAcceptEncoding; > + } else if (token == "enabled" ) { > + state = kParseEnable; > + } else if (token == "cache" ) { > + state = kParseCache; > + } else if (token == "disallow" ) { > + state = kParseDisallow; > + } > + else { > + warning("failed to interpret \"%s\" at line %zu", token.c_str(), > lineno); > + } > + break; > + case kParseCompressibleContentType: > + current_host_configuration->add_compressible_content_type(token); > + state = kParseStart; > + break; > + case kParseRemoveAcceptEncoding: > + current_host_configuration->set_remove_accept_encoding(token == > "true"); > + state = kParseStart; > + break; > + case kParseEnable: > + current_host_configuration->set_enabled(token == "true"); > + state = kParseStart; > + break; > + case kParseCache: > + current_host_configuration->set_cache(token == "true"); > + state = kParseStart; > + break; > + case kParseDisallow: > + current_host_configuration->add_disallow(token); > + state = kParseStart; > + break; > + } > + } > + } > + > + if (state != kParseStart) { > + warning("the parser state indicates that data was expected when it > reached the end of the file (%d)", state); > + } > + > + return c; > + } //Configuration::Parse > +} //namespace > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/configuration.h > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/configuration.h > b/plugins/experimental/ats_speed/gzip/configuration.h > new file mode 100644 > index 0000000..b38cb64 > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/configuration.h > @@ -0,0 +1,84 @@ > +/** @file > + > + Transforms content using gzip or deflate > + > + @section license License > + > + Licensed to the Apache Software Foundation (ASF) under one > + or more contributor license agreements. See the NOTICE file > + distributed with this work for additional information > + regarding copyright ownership. The ASF licenses this file > + to you under the Apache License, Version 2.0 (the > + "License"); you may not use this file except in compliance > + with the License. You may obtain a copy of the License at > + > + http://www.apache.org/licenses/LICENSE-2.0 > + > + Unless required by applicable law or agreed to in writing, software > + distributed under the License is distributed on an "AS IS" BASIS, > + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + See the License for the specific language governing permissions and > + limitations under the License. > + */ > + > + > +#ifndef GZIP_CONFIGURATION_H_ > +#define GZIP_CONFIGURATION_H_ > + > +#include <string> > +#include <vector> > +#include "debug_macros.h" > + > +namespace Gzip { > + class HostConfiguration { > + public: //todo -> only configuration should be able to construct hostconfig > + explicit HostConfiguration(const std::string & host) > + : host_(host) > + , enabled_(true) > + , cache_(true) > + , remove_accept_encoding_(false) > + {} > + > + inline bool enabled() { return enabled_; } > + inline void set_enabled(bool x) { enabled_ = x; } > + inline bool cache() { return cache_; } > + inline void set_cache(bool x) { cache_ = x; } > + inline bool remove_accept_encoding() { return remove_accept_encoding_; } > + inline void set_remove_accept_encoding(bool x) { remove_accept_encoding_ > = x; } > + inline std::string host() { return host_; } > + void add_disallow(const std::string & disallow); > + void add_compressible_content_type(const std::string & content_type); > + bool IsUrlAllowed(const char * url, int url_len); > + bool ContentTypeIsCompressible(const char * content_type, int > content_type_length); > + > + private: > + std::string host_; > + bool enabled_; > + bool cache_; > + bool remove_accept_encoding_; > + std::vector<std::string> compressible_content_types_; > + std::vector<std::string> disallows_; > + DISALLOW_COPY_AND_ASSIGN(HostConfiguration); > + };//class HostConfiguration > + > + class Configuration { > + friend class HostConfiguration; > + public: > + static Configuration * Parse(const char * path); > + HostConfiguration * Find(const char * host, int host_length); > + inline HostConfiguration * GlobalConfiguration() { > + return host_configurations_[0]; > + } > + > + private: > + explicit Configuration() {} > + void AddHostConfiguration(HostConfiguration * hc); > + > + std::vector<HostConfiguration *> host_configurations_; > + //todo: destructor. delete owned host configurations > + DISALLOW_COPY_AND_ASSIGN(Configuration); > + }; //class Configuration > + > +}//namespace > + > +#endif > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/debug_macros.h > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/debug_macros.h > b/plugins/experimental/ats_speed/gzip/debug_macros.h > new file mode 100644 > index 0000000..151de31 > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/debug_macros.h > @@ -0,0 +1,59 @@ > +/** @file > + > + Transforms content using gzip or deflate > + > + @section license License > + > + Licensed to the Apache Software Foundation (ASF) under one > + or more contributor license agreements. See the NOTICE file > + distributed with this work for additional information > + regarding copyright ownership. The ASF licenses this file > + to you under the Apache License, Version 2.0 (the > + "License"); you may not use this file except in compliance > + with the License. You may obtain a copy of the License at > + > + http://www.apache.org/licenses/LICENSE-2.0 > + > + Unless required by applicable law or agreed to in writing, software > + distributed under the License is distributed on an "AS IS" BASIS, > + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + See the License for the specific language governing permissions and > + limitations under the License. > + */ > + > +#ifndef _DBG_MACROS_H > +#define _DBG_MACROS_H > + > +#include <ts/ts.h> > + > +#define TAG "gzip" > + > +#define debug(fmt, args...) do { \ > + TSDebug(TAG, "DEBUG: [%s:%d] [%s] " fmt, __FILE__, __LINE__, __FUNCTION__ > , ##args ); \ > + } while (0) > + > +#define info(fmt, args...) do { \ > + TSDebug(TAG, "INFO: " fmt, ##args ); \ > + } while (0) > + > +#define warning(fmt, args...) do { \ > + TSDebug(TAG, "WARNING: " fmt, ##args ); \ > +} while (0) > + > +#define error(fmt, args...) do { \ > + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , > ##args ); \ > + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ > , ##args ); \ > +} while (0) > + > +#define fatal(fmt, args...) do { \ > + TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , > ##args ); \ > + TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ > , ##args ); \ > + exit(-1); \ > +} while (0) > + > +//FIXME: this one doesn't deserve to be here > +#define DISALLOW_COPY_AND_ASSIGN(TypeName) \ > + TypeName(const TypeName&); \ > + void operator=(const TypeName&) > + > +#endif //_DBG_MACROS_H > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.cc > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/gzip.cc > b/plugins/experimental/ats_speed/gzip/gzip.cc > new file mode 100644 > index 0000000..1397762 > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/gzip.cc > @@ -0,0 +1,826 @@ > +/** @file > + > + Transforms content using gzip or deflate > + > + @section license License > + > + Licensed to the Apache Software Foundation (ASF) under one > + or more contributor license agreements. See the NOTICE file > + distributed with this work for additional information > + regarding copyright ownership. The ASF licenses this file > + to you under the Apache License, Version 2.0 (the > + "License"); you may not use this file except in compliance > + with the License. You may obtain a copy of the License at > + > + http://www.apache.org/licenses/LICENSE-2.0 > + > + Unless required by applicable law or agreed to in writing, software > + distributed under the License is distributed on an "AS IS" BASIS, > + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + See the License for the specific language governing permissions and > + limitations under the License. > + */ > +#define __STDC_LIMIT_MACROS > +#define __STDC_CONSTANT_MACROS > +#include <stdint.h> > + > +#include <string> > +#include <string.h> > +#include <zlib.h> > +#include <ts/ts.h> > +#include "debug_macros.h" > +#include "misc.h" > +#include "configuration.h" > +//#include "ink_defs.h" > + > +using namespace std; > +using namespace Gzip; > + > +//FIXME: custom dictionaries would be nice. configurable/content-type? > +//FIXME: look into autoscaling the compression level based on connection > speed > +// a gprs device might benefit from a higher compression ratio, whereas a > desktop w. high bandwith > +// might be served better with little or no compression at all > +//FIXME: look into compressing from the task thread pool > +//FIXME: make normalizing accept encoding configurable > + > +// from mod_deflate: > +// ZLIB's compression algorithm uses a > +// 0-9 based scale that GZIP does where '1' is 'Best speed' > +// and '9' is 'Best compression'. Testing has proved level '6' > +// to be about the best level to use in an HTTP Server. > + > +const int ZLIB_COMPRESSION_LEVEL = 6; > + > +int arg_idx_hooked; > +int arg_idx_host_configuration; > +int arg_idx_url_disallowed; > + > + > +const char * global_hidden_header_name; > +Configuration* config = NULL; > +const char *dictionary = NULL; > + > +static GzipData * > +gzip_data_alloc(int compression_type) > +{ > + GzipData *data; > + int err; > + > + data = (GzipData *) TSmalloc(sizeof(GzipData)); > + data->downstream_vio = NULL; > + data->downstream_buffer = NULL; > + data->downstream_reader = NULL; > + data->downstream_length = 0; > + data->state = transform_state_initialized; > + data->compression_type = compression_type; > + data->zstrm.next_in = Z_NULL; > + data->zstrm.avail_in = 0; > + data->zstrm.total_in = 0; > + data->zstrm.next_out = Z_NULL; > + data->zstrm.avail_out = 0; > + data->zstrm.total_out = 0; > + data->zstrm.zalloc = gzip_alloc; > + data->zstrm.zfree = gzip_free; > + data->zstrm.opaque = (voidpf) 0; > + data->zstrm.data_type = Z_ASCII; > + > + int window_bits = (compression_type == COMPRESSION_TYPE_GZIP) ? > WINDOW_BITS_GZIP : WINDOW_BITS_DEFLATE; > + > + err = deflateInit2(&data->zstrm, ZLIB_COMPRESSION_LEVEL, Z_DEFLATED, > window_bits, ZLIB_MEMLEVEL, Z_DEFAULT_STRATEGY); > + > + if (err != Z_OK) { > + fatal("gzip-transform: ERROR: deflateInit (%d)!", err); > + } > + > + if (dictionary) { > + err = deflateSetDictionary(&data->zstrm, (const Bytef *) dictionary, > strlen(dictionary)); > + if (err != Z_OK) { > + fatal("gzip-transform: ERROR: deflateSetDictionary (%d)!", err); > + } > + } > + > + return data; > +} > + > + > +static void > +gzip_data_destroy(GzipData * data) > +{ > + TSReleaseAssert(data); > + > + //deflateEnd returnvalue ignore is intentional > + //it would spew log on every client abort > + deflateEnd(&data->zstrm); > + > + if (data->downstream_buffer) { > + TSIOBufferDestroy(data->downstream_buffer); > + } > + > + TSfree(data); > +} > + > +static TSReturnCode > +gzip_content_encoding_header(TSMBuffer bufp, TSMLoc hdr_loc, const int > compression_type) > +{ > + TSReturnCode ret; > + TSMLoc ce_loc; > + > + // Delete Content-Encoding if present??? > + > + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Content-Encoding", > sizeof("Content-Encoding") - 1, &ce_loc)) == TS_SUCCESS) { > + if (compression_type == COMPRESSION_TYPE_DEFLATE) { > + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, > "deflate", sizeof("deflate") - 1); > + } else if (compression_type == COMPRESSION_TYPE_GZIP) { > + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, > "gzip", sizeof("gzip") - 1); > + } > + if (ret == TS_SUCCESS) { > + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc); > + } > + TSHandleMLocRelease(bufp, hdr_loc, ce_loc); > + } > + > + if (ret != TS_SUCCESS) { > + error("cannot add the Content-Encoding header"); > + } > + > + return ret; > +} > + > +static TSReturnCode > +gzip_vary_header(TSMBuffer bufp, TSMLoc hdr_loc) > +{ > + TSReturnCode ret; > + TSMLoc ce_loc; > + > + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "Vary", sizeof("Vary") - 1); > + if (ce_loc) { > + int idx, count, len; > + const char *value; > + > + count = TSMimeHdrFieldValuesCount(bufp, hdr_loc, ce_loc); > + for(idx=0; idx<count; idx++) { > + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, idx, &len); > + if (len && > + strncasecmp("Accept-Encoding", value, len) == 0) { > + // Bail, Vary: Accept-Encoding already sent from origin > + TSHandleMLocRelease(bufp, hdr_loc, ce_loc); > + return TS_SUCCESS; > + } > + } > + > + ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, > "Accept-Encoding", sizeof("Accept-Encoding") - 1); > + TSHandleMLocRelease(bufp, hdr_loc, ce_loc); > + } else { > + if ((ret = TSMimeHdrFieldCreateNamed(bufp, hdr_loc, "Vary", > sizeof("Vary") - 1, &ce_loc)) == TS_SUCCESS) { > + if ((ret = TSMimeHdrFieldValueStringInsert(bufp, hdr_loc, ce_loc, -1, > "Accept-Encoding", sizeof("Accept-Encoding") - 1)) == TS_SUCCESS) { > + ret = TSMimeHdrFieldAppend(bufp, hdr_loc, ce_loc); > + } > + > + TSHandleMLocRelease(bufp, hdr_loc, ce_loc); > + } > + } > + > + if (ret != TS_SUCCESS) { > + error("cannot add/update the Vary header"); > + } > + > + return ret; > +} > + > +//FIXME: the etag alteration isn't proper. it should modify the value inside > quotes > +// specify a very header.. > +static TSReturnCode > +gzip_etag_header(TSMBuffer bufp, TSMLoc hdr_loc) > +{ > + TSReturnCode ret = TS_SUCCESS; > + TSMLoc ce_loc; > + > + ce_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_ETAG, > TS_MIME_LEN_ETAG); > + > + if (ce_loc) { > + int changetag = 1; > + int strl; > + const char *strv = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, ce_loc, > -1, &strl); > + //do not alter weak etags. > + //FIXME: consider just making the etag weak for compressed content > + if (strl >= 2) { > + if ((strv[0] == 'w' || strv[0] == 'W') && strv[1] == '/') { > + changetag = 0; > + } > + if (changetag) { > + ret = TSMimeHdrFieldValueAppend(bufp, hdr_loc, ce_loc, 0, "-df", 3); > + } > + } > + TSHandleMLocRelease(bufp, hdr_loc, ce_loc); > + } > + > + if (ret != TS_SUCCESS) { > + error("cannot handle the %s header", TS_MIME_FIELD_ETAG); > + } > + > + return ret; > +} > + > +//FIXME: some things are potentially compressible. those responses > +static void > +gzip_transform_init(TSCont contp, GzipData * data) > +{ > + //update the vary, content-encoding, and etag response headers > + //prepare the downstream for transforming > + > + TSVConn downstream_conn; > + TSMBuffer bufp; > + TSMLoc hdr_loc; > + > + data->state = transform_state_output; > + > + if (TSHttpTxnTransformRespGet(data->txn, &bufp, &hdr_loc) != TS_SUCCESS) { > + error("Error TSHttpTxnTransformRespGet"); > + return; > + } > + > + if (gzip_content_encoding_header(bufp, hdr_loc, data->compression_type) == > TS_SUCCESS && > + gzip_vary_header(bufp, hdr_loc) == TS_SUCCESS && > + gzip_etag_header(bufp, hdr_loc) == TS_SUCCESS) { > + downstream_conn = TSTransformOutputVConnGet(contp); > + data->downstream_buffer = TSIOBufferCreate(); > + data->downstream_reader = TSIOBufferReaderAlloc(data->downstream_buffer); > + data->downstream_vio = TSVConnWrite(downstream_conn, contp, > data->downstream_reader, INT64_MAX); > + } > + > + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); > +} > + > + > + > +static void > +gzip_transform_one(GzipData * data, TSIOBufferReader upstream_reader, int > amount) > +{ > + TSIOBufferBlock downstream_blkp; > + const char *upstream_buffer; > + char *downstream_buffer; > + int64_t upstream_length, downstream_length; > + int err; > + > + while (amount > 0) { > + downstream_blkp = TSIOBufferReaderStart(upstream_reader); > + if (!downstream_blkp) { > + error("couldn't get from IOBufferBlock"); > + return; > + } > + > + upstream_buffer = TSIOBufferBlockReadStart(downstream_blkp, > upstream_reader, &upstream_length); > + if (!upstream_buffer) { > + error("couldn't get from TSIOBufferBlockReadStart"); > + return; > + } > + > + if (upstream_length > amount) { > + upstream_length = amount; > + } > + > + data->zstrm.next_in = (unsigned char *) upstream_buffer; > + data->zstrm.avail_in = upstream_length; > + > + while (data->zstrm.avail_in > 0) { > + downstream_blkp = TSIOBufferStart(data->downstream_buffer); > + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, > &downstream_length); > + > + data->zstrm.next_out = (unsigned char *) downstream_buffer; > + data->zstrm.avail_out = downstream_length; > + > + err = deflate(&data->zstrm, Z_NO_FLUSH); > + > + if (err != Z_OK) > + warning("deflate() call failed: %d", err); > + > + if (downstream_length > data->zstrm.avail_out) { > + TSIOBufferProduce(data->downstream_buffer, downstream_length - > data->zstrm.avail_out); > + data->downstream_length += (downstream_length - > data->zstrm.avail_out); > + } > + > + if (data->zstrm.avail_out > 0) { > + if (data->zstrm.avail_in != 0) { > + error("gzip-transform: ERROR: avail_in is (%d): should be 0", > data->zstrm.avail_in); > + } > + } > + } > + > + TSIOBufferReaderConsume(upstream_reader, upstream_length); > + amount -= upstream_length; > + } > +} > + > +static void > +gzip_transform_finish(GzipData * data) > +{ > + if (data->state == transform_state_output) { > + TSIOBufferBlock downstream_blkp; > + char *downstream_buffer; > + int64_t downstream_length; > + int err; > + > + data->state = transform_state_finished; > + > + for (;;) { > + downstream_blkp = TSIOBufferStart(data->downstream_buffer); > + > + downstream_buffer = TSIOBufferBlockWriteStart(downstream_blkp, > &downstream_length); > + data->zstrm.next_out = (unsigned char *) downstream_buffer; > + data->zstrm.avail_out = downstream_length; > + > + err = deflate(&data->zstrm, Z_FINISH); > + > + if (downstream_length > (int64_t) data->zstrm.avail_out) { > + TSIOBufferProduce(data->downstream_buffer, downstream_length - > data->zstrm.avail_out); > + data->downstream_length += (downstream_length - > data->zstrm.avail_out); > + } > + > + if (err == Z_OK) { /* some more data to encode */ > + continue; > + } > + > + if (err != Z_STREAM_END) { > + warning("deflate should report Z_STREAM_END"); > + } > + break; > + } > + > + if (data->downstream_length != (int64_t) (data->zstrm.total_out)) { > + error("gzip-transform: ERROR: output lengths don't match (%d, %ld)", > data->downstream_length, > + data->zstrm.total_out); > + } > + > + gzip_log_ratio(data->zstrm.total_in, data->downstream_length); > + } > +} > + > + > +static void > +gzip_transform_do(TSCont contp) > +{ > + TSVIO upstream_vio; > + GzipData *data; > + int64_t upstream_todo; > + int64_t upstream_avail; > + int64_t downstream_bytes_written; > + > + data = (GzipData*)TSContDataGet(contp); > + if (data->state == transform_state_initialized) { > + gzip_transform_init(contp, data); > + } > + > + upstream_vio = TSVConnWriteVIOGet(contp); > + downstream_bytes_written = data->downstream_length; > + > + if (!TSVIOBufferGet(upstream_vio)) { > + gzip_transform_finish(data); > + > + TSVIONBytesSet(data->downstream_vio, data->downstream_length); > + > + if (data->downstream_length > downstream_bytes_written) { > + TSVIOReenable(data->downstream_vio); > + } > + return; > + } > + > + upstream_todo = TSVIONTodoGet(upstream_vio); > + > + if (upstream_todo > 0) { > + upstream_avail = TSIOBufferReaderAvail(TSVIOReaderGet(upstream_vio)); > + > + if (upstream_todo > upstream_avail) { > + upstream_todo = upstream_avail; > + } > + > + if (upstream_todo > 0) { > + gzip_transform_one(data, TSVIOReaderGet(upstream_vio), upstream_todo); > + TSVIONDoneSet(upstream_vio, TSVIONDoneGet(upstream_vio) + > upstream_todo); > + } > + } > + > + if (TSVIONTodoGet(upstream_vio) > 0) { > + if (upstream_todo > 0) { > + if (data->downstream_length > downstream_bytes_written) { > + TSVIOReenable(data->downstream_vio); > + } > + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_READY, > upstream_vio); > + } > + } else { > + gzip_transform_finish(data); > + TSVIONBytesSet(data->downstream_vio, data->downstream_length); > + > + if (data->downstream_length > downstream_bytes_written) { > + TSVIOReenable(data->downstream_vio); > + } > + > + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_VCONN_WRITE_COMPLETE, > upstream_vio); > + } > +} > + > + > +static int > +gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) > +{ > + if (TSVConnClosedGet(contp)) { > + gzip_data_destroy((GzipData*)TSContDataGet(contp)); > + TSContDestroy(contp); > + return 0; > + } else { > + switch (event) { > + case TS_EVENT_ERROR:{ > + debug("gzip_transform: TS_EVENT_ERROR starts"); > + TSVIO upstream_vio = TSVConnWriteVIOGet(contp); > + TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio); > + } > + break; > + case TS_EVENT_VCONN_WRITE_COMPLETE: > + TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1); > + break; > + case TS_EVENT_VCONN_WRITE_READY: > + gzip_transform_do(contp); > + break; > + case TS_EVENT_IMMEDIATE: > + gzip_transform_do(contp); > + break; > + default: > + warning("unknown event [%d]", event); > + gzip_transform_do(contp); > + break; > + } > + } > + > + return 0; > +} > + > + > +static int > +gzip_transformable(TSHttpTxn txnp, int server, HostConfiguration * > host_configuration, int *compress_type) > +{ > + /* Server response header */ > + TSMBuffer bufp; > + TSMLoc hdr_loc; > + TSMLoc field_loc; > + > + /* Client request header */ > + TSMBuffer cbuf; > + TSMLoc chdr; > + TSMLoc cfield; > + > + const char *value; > + int nvalues; > + int i, compression_acceptable, len; > + > + TSHttpStatus resp_status; > + if (server) { > + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc); > + } else { > + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc); > + } > + resp_status = TSHttpHdrStatusGet(bufp, hdr_loc); > + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); > + > + //conservatively pick some statusses to compress > + if (!(resp_status == 200 || resp_status == 404 || resp_status == 500)) { > + info("http response status [%d] is not compressible", resp_status); > + return 0; > + } > + > + TSHttpTxnClientReqGet(txnp, &cbuf, &chdr); > + > + //the only compressible method is currently GET. > + int method_length; > + const char *method = TSHttpHdrMethodGet(cbuf, chdr, &method_length); > + if (!(method_length == TS_HTTP_LEN_GET && memcmp(method, > TS_HTTP_METHOD_GET, TS_HTTP_LEN_GET) == 0)) { > + debug("method is not GET, not compressible"); > + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr); > + return 0; > + } > + > + cfield = TSMimeHdrFieldFind(cbuf, chdr, TS_MIME_FIELD_ACCEPT_ENCODING, > TS_MIME_LEN_ACCEPT_ENCODING); > + if (cfield != TS_NULL_MLOC) { > + compression_acceptable = 0; > + nvalues = TSMimeHdrFieldValuesCount(cbuf, chdr, cfield); > + for (i=0; i<nvalues; i++) { > + value = TSMimeHdrFieldValueStringGet(cbuf, chdr, cfield, i, &len); > + if (!value) { > + continue; > + } > + > + if (strncasecmp(value, "deflate", sizeof("deflate") - 1) == 0) { > + compression_acceptable = 1; > + *compress_type = COMPRESSION_TYPE_DEFLATE; > + break; > + } else if (strncasecmp(value, "gzip", sizeof("gzip") - 1) == 0) { > + compression_acceptable = 1; > + *compress_type = COMPRESSION_TYPE_GZIP; > + break; > + } > + } > + > + TSHandleMLocRelease(cbuf, chdr, cfield); > + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr); > + > + if (!compression_acceptable) { > + info("no acceptable encoding found in request header, not > compressible"); > + return 0; > + } > + } else { > + info("no acceptable encoding found in request header, not compressible"); > + TSHandleMLocRelease(cbuf, chdr, cfield); > + TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr); > + return 0; > + } > + > + if (server) { > + TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc); > + } else { > + TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc); > + } > + > + /* If there already exists a content encoding then we don't want > + to do anything. */ > + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, > TS_MIME_FIELD_CONTENT_ENCODING, -1); > + if (field_loc) { > + info("response is already content encoded, not compressible"); > + TSHandleMLocRelease(bufp, hdr_loc, field_loc); > + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); > + return 0; > + } > + > + /* We only want to do gzip compression on documents that have a > + content type of "text/" or "application/x-javascript". */ > + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_TYPE, > -1); > + if (!field_loc) { > + info("no content type header found, not compressible"); > + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); > + return 0; > + } > + > + value = TSMimeHdrFieldValueStringGet(bufp, hdr_loc, field_loc, 0, &len); > + > + int rv = host_configuration->ContentTypeIsCompressible(value, len); > + if (!rv) { > + info("content-type [%.*s] not compressible", len, value); > + } > + TSHandleMLocRelease(bufp, hdr_loc, field_loc); > + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); > + return rv; > +} > + > + > +static void > +gzip_transform_add(TSHttpTxn txnp, int /* server ATS_UNUSED */, > HostConfiguration * hc, int compress_type) > +{ > + int *tmp = (int *) TSHttpTxnArgGet(txnp, arg_idx_hooked); > + if (tmp) { > + //happens on cache_stale_hit > + debug("transform hook already set, bail"); > + return; > + } else { > + TSHttpTxnArgSet(txnp, arg_idx_hooked, (void *) &GZIP_ONE); > + info("adding compression transform"); > + } > + > + TSHttpTxnUntransformedRespCache(txnp, 1); > + > + if (!hc->cache()) { > + debug("@@@@@ Gzip cache disabled"); > + TSHttpTxnTransformedRespCache(txnp, 0); > + } else { > + TSMBuffer bufp; > + TSMLoc hdr_loc, field_loc; > + int cache = 1; > + if (TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS || > TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc) == TS_SUCCESS) { > + field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, "@gzip_nocache", > strlen("@gzip_nocache")); > + if (field_loc) { > + cache = 0; > + debug("@@@@@ Gzip disallows cacheing of transformed response"); > + TSHandleMLocRelease(bufp, hdr_loc, field_loc); > + } else { > + debug("@@@@ Gzip allows cacheing of transformed response"); > + } > + TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc); > + } > + > + TSHttpTxnTransformedRespCache(txnp, cache); > + } > + > + TSVConn connp; > + GzipData *data; > + > + connp = TSTransformCreate(gzip_transform, txnp); > + data = gzip_data_alloc(compress_type); > + data->txn = txnp; > + > + TSContDataSet(connp, data); > + TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp); > +} > + > +static int > +cache_transformable(TSHttpTxn txnp) > +{ > + int obj_status; > + > + if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) { > + warning("Couldn't get cache status of object"); > + return 0; > + } > + if (obj_status == TS_CACHE_LOOKUP_HIT_STALE) { > + info("stale cache hit"); > + return 0; > + } > + if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) { > + info("fresh cache hit"); > + return 1; > + } > + > + return 0; > +} > + > +HostConfiguration * > +find_host_configuration(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer bufp, > TSMLoc locp) > +{ > + TSMLoc fieldp = TSMimeHdrFieldFind(bufp, locp, TS_MIME_FIELD_HOST, > TS_MIME_LEN_HOST); > + > + if (fieldp) { > + int strl; > + const char *strv = TSMimeHdrFieldValueStringGet(bufp, locp, fieldp, -1, > &strl); > + TSHandleMLocRelease(bufp, locp, fieldp); > + > + HostConfiguration * host_configuration = config->Find(strv, strl); > + return host_configuration; > + } > + > + return config->GlobalConfiguration(); > +} > + > + > +static int > +transform_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata) > +{ > + TSHttpTxn txnp = (TSHttpTxn) edata; > + int compress_type = COMPRESSION_TYPE_DEFLATE; > + > + switch (event) { > + case TS_EVENT_HTTP_READ_REQUEST_HDR: > + { > + TSMBuffer req_buf; > + TSMLoc req_loc; > + if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) { > + int url_len; > + char * url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len); > + HostConfiguration * hc = find_host_configuration(txnp, req_buf, > req_loc); > + //we could clone the hosting configuration here, to make it > deletable on reload? > + TSHttpTxnArgSet(txnp, arg_idx_host_configuration, (void *) hc); > + > + if (!hc->enabled() || !hc->IsUrlAllowed(url, url_len)) { > + //FIXME: no double negatives > + TSHttpTxnArgSet(txnp, arg_idx_url_disallowed, (void *) > &GZIP_ONE); > + info("url [%.*s] not allowed", url_len, url); > + } else { > + normalize_accept_encoding(txnp, req_buf, req_loc); > + } > + TSfree(url); > + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc); > + } > + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); > + } > + break; > + > + case TS_EVENT_HTTP_READ_RESPONSE_HDR: > + { > + //os: the accept encoding header needs to be restored.. > + //otherwise the next request won't get a cache hit on this > + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, > arg_idx_host_configuration); > + if (hc != NULL) { > + if (hc->remove_accept_encoding()) { > + TSMBuffer req_buf; > + TSMLoc req_loc; > + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == > TS_SUCCESS) { > + restore_accept_encoding(txnp, req_buf, req_loc, > global_hidden_header_name); > + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc); > + } > + } > + > + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed); > + if ( allowed && gzip_transformable(txnp, 1, hc, &compress_type)) { > + gzip_transform_add(txnp, 1, hc, compress_type); > + } > + } > + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); > + } > + break; > + > + case TS_EVENT_HTTP_SEND_REQUEST_HDR: > + { > + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, > arg_idx_host_configuration); > + if (hc!=NULL) { > + if (hc->remove_accept_encoding()) { > + TSMBuffer req_buf; > + TSMLoc req_loc; > + if (TSHttpTxnServerReqGet(txnp, &req_buf, &req_loc) == > TS_SUCCESS) { > + hide_accept_encoding(txnp, req_buf, req_loc, > global_hidden_header_name); > + TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc); > + } > + } > + } > + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); > + } > + break; > + > + case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE: > + { > + int allowed = !TSHttpTxnArgGet(txnp, arg_idx_url_disallowed); > + HostConfiguration * hc = (HostConfiguration*)TSHttpTxnArgGet(txnp, > arg_idx_host_configuration); > + if ( hc != NULL ) { > + if (allowed && cache_transformable(txnp) && > gzip_transformable(txnp, 0, hc, &compress_type)) { > + gzip_transform_add(txnp, 0, hc, compress_type); > + } > + } > + TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE); > + } > + break; > + > + default: > + fatal("gzip transform unknown event"); > + } > + > + return 0; > +} > + > + > +static void > +read_configuration(TSCont contp) { > + const char * path = (const char *)TSContDataGet(contp); > + Configuration * newconfig = Configuration::Parse(path); > + > + Configuration * oldconfig =__sync_lock_test_and_set(&config, newconfig); > + debug("config swapped,old config %p", oldconfig); > + > + //FIXME: we have leaked. > + //consider cloning or refcounting the configuration passed to the txn > + //to make deleting the old configuration possible > + //if (config != NULL ) > + // delete config; > +} > + > +static int > +management_update(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */) > +{ > + TSReleaseAssert(event == TS_EVENT_MGMT_UPDATE); > + info("management update event received"); > + read_configuration(contp); > + return 0; > +} > + > + > +void > +TSPluginInit(int argc, const char *argv[]) > +{ > + string config_path; > + > + if (argc > 2) { > + fatal("the gzip plugin does not accept more than 1 plugin argument"); > + } else if (argc == 2) { > + config_path = std::string(argv[1]); > + } > + > + info("TSPluginInit %s", argv[0]); > + > + if (!register_plugin()) { > + fatal("The gzip plugin failed to register"); > + } > + > + //if (argc == 2) { > + // dictionary = load_dictionary(argv[1]); > + //} > + > + if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set", > &arg_idx_hooked) != TS_SUCCESS) { > + fatal("failed to reserve an argument index"); > + } > + if (TSHttpArgIndexReserve("gzip", "for storing if compression is > applicable", &arg_idx_host_configuration) != TS_SUCCESS) { > + fatal("failed to reserve an argument index"); > + } > + if (TSHttpArgIndexReserve("gzip", "for storing if compression is > disallowed for this txn", &arg_idx_url_disallowed) != TS_SUCCESS) { > + fatal("failed to reserve an argument index"); > + } > + > + global_hidden_header_name = init_hidden_header_name(); > + > + TSCont management_contp = TSContCreate(management_update, NULL); > + //fixme: never freed. there is no shutdown event? > + char * p = (char*)TSmalloc(config_path.size()+1); > + strcpy(p,config_path.c_str()); > + TSContDataSet(management_contp,(void*)p); > + TSMgmtUpdateRegister(management_contp, TAG); > + read_configuration(management_contp); > + > + TSCont transform_contp = TSContCreate(transform_plugin, NULL); > + TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp); > + TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp); > + TSHttpHookAdd(TS_HTTP_SEND_REQUEST_HDR_HOOK, transform_contp); > + TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, transform_contp); > + > + info("loaded"); > +} > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/gzip.config > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/gzip.config > b/plugins/experimental/ats_speed/gzip/gzip.config > new file mode 100644 > index 0000000..81e9fa4 > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/gzip.config > @@ -0,0 +1,6 @@ > +# Set some global options first > +cache true > +enabled true > +remove-accept-encoding false > +compressible-content-type text/* > +compressible-content-type *javascript* > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.cc > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/misc.cc > b/plugins/experimental/ats_speed/gzip/misc.cc > new file mode 100644 > index 0000000..0ea6911 > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/misc.cc > @@ -0,0 +1,197 @@ > +/** @file > + > + Transforms content using gzip or deflate > + > + @section license License > + > + Licensed to the Apache Software Foundation (ASF) under one > + or more contributor license agreements. See the NOTICE file > + distributed with this work for additional information > + regarding copyright ownership. The ASF licenses this file > + to you under the Apache License, Version 2.0 (the > + "License"); you may not use this file except in compliance > + with the License. You may obtain a copy of the License at > + > + http://www.apache.org/licenses/LICENSE-2.0 > + > + Unless required by applicable law or agreed to in writing, software > + distributed under the License is distributed on an "AS IS" BASIS, > + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + See the License for the specific language governing permissions and > + limitations under the License. > + */ > + > +#include "ts/ts.h" > +//#include "ink_defs.h" > +#include <stdint.h> > +#include <inttypes.h> > +#include "misc.h" > +#include <string.h> > +#include "debug_macros.h" > + > +voidpf > +gzip_alloc(voidpf /* opaque ATS_UNUSED */, uInt items, uInt size) > +{ > + return (voidpf) TSmalloc(items * size); > +} > + > +void > +gzip_free(voidpf /* opaque ATS_UNUSED */, voidpf address) > +{ > + TSfree(address); > +} > + > +void > +normalize_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, > TSMLoc hdr_loc) > +{ > + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, > TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); > + int deflate = 0; > + int gzip = 0; > + > + //remove the accept encoding field(s), > + //while finding out if gzip or deflate is supported. > + while (field) { > + TSMLoc tmp; > + > + if (!deflate && !gzip) { > + int value_count = TSMimeHdrFieldValuesCount(reqp, hdr_loc, field); > + > + while (value_count > 0) { > + int val_len = 0; > + const char *val; > + > + --value_count; > + val = TSMimeHdrFieldValueStringGet(reqp, hdr_loc, field, > value_count, &val_len); > + > + if (val_len == (int) strlen("gzip")) > + gzip = !strncmp(val, "gzip", val_len); > + else if (val_len == (int) strlen("deflate")) > + deflate = !strncmp(val, "deflate", val_len); > + } > + } > + > + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field); > + TSMimeHdrFieldDestroy(reqp, hdr_loc, field); //catch retval? > + TSHandleMLocRelease(reqp, hdr_loc, field); > + field = tmp; > + } > + > + //append a new accept-encoding field in the header > + if (deflate || gzip) { > + TSMimeHdrFieldCreate(reqp, hdr_loc, &field); > + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, > TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); > + > + if (gzip) { > + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "gzip", > strlen("gzip")); > + info("normalized accept encoding to gzip"); > + } else if (deflate) { > + TSMimeHdrFieldValueStringInsert(reqp, hdr_loc, field, -1, "deflate", > strlen("deflate")); > + info("normalized accept encoding to deflate"); > + } > + > + TSMimeHdrFieldAppend(reqp, hdr_loc, field); > + TSHandleMLocRelease(reqp, hdr_loc, field); > + } > +} > + > +void > +hide_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, TSMLoc > hdr_loc, const char * hidden_header_name) > +{ > + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, > TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); > + while (field) { > + TSMLoc tmp; > + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field); > + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, hidden_header_name, -1); > + TSHandleMLocRelease(reqp, hdr_loc, field); > + field = tmp; > + } > +} > + > +void > +restore_accept_encoding(TSHttpTxn /* txnp ATS_UNUSED */, TSMBuffer reqp, > TSMLoc hdr_loc, const char * hidden_header_name) > +{ > + TSMLoc field = TSMimeHdrFieldFind(reqp, hdr_loc, hidden_header_name, -1); > + > + while (field) { > + TSMLoc tmp; > + tmp = TSMimeHdrFieldNextDup(reqp, hdr_loc, field); > + TSMimeHdrFieldNameSet(reqp, hdr_loc, field, > TS_MIME_FIELD_ACCEPT_ENCODING, TS_MIME_LEN_ACCEPT_ENCODING); > + TSHandleMLocRelease(reqp, hdr_loc, field); > + field = tmp; > + } > +} > + > +const char * > +init_hidden_header_name() > +{ > + char * hidden_header_name; > + const char *var_name = "proxy.config.proxy_name"; > + TSMgmtString result; > + > + if (TSMgmtStringGet(var_name, &result) != TS_SUCCESS) { > + fatal("failed to get server name"); > + } else { > + int hidden_header_name_len = strlen("x-accept-encoding-") + > strlen(result); > + hidden_header_name = (char *) TSmalloc(hidden_header_name_len + 1); > + hidden_header_name[hidden_header_name_len] = 0; > + sprintf(hidden_header_name, "x-accept-encoding-%s", result); > + } > + return hidden_header_name; > +} > + > +int > +register_plugin() > +{ > + TSPluginRegistrationInfo info; > + > + info.plugin_name = (char*)"gzip"; > + info.vendor_name = (char*)"Apache"; > + info.support_email = (char*)"dev@trafficserver.apache.org"; > + > + if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) { > + return 0; > + } > + return 1; > +} > + > +const char * > +load_dictionary(const char *preload_file) > +{ > + char *dict = (char *) malloc(800000); > + uLong dictId = adler32(0L, Z_NULL, 0); > + uLong *adler = &dictId; > + > + FILE *fp; > + int i = 0; > + > + fp = fopen(preload_file, "r"); > + if (!fp) { > + fatal("gzip-transform: ERROR: Unable to open dict file %s", > preload_file); > + } > + > + /* dict = (char *) calloc(8000, sizeof(char)); */ > + > + i = 0; > + while (!feof(fp)) { > + if (fscanf(fp, "%s\n", dict + i) == 1) { > + i = strlen(dict); > + strcat(dict + i, " "); > + ++i; > + } > + } > + dict[i - 1] = '\0'; > + > + /* TODO get the adler compute right */ > + *adler = adler32(*adler, (const Byte *) dict, sizeof(dict)); > + return dict; > +} > + > +void > +gzip_log_ratio(int64_t in, int64_t out) > +{ > + // if (in) { > + // info("Compressed size %PRId64 (bytes), Original size %" PRId64", > ratio: %f", out, in, ((float) (in - out) / in)); > + //} else { > + // debug("Compressed size %PRId64 (bytes), Original size %" PRId64", > ratio: %f", out, in, 0.0F); > + // } > +} > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/gzip/misc.h > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/gzip/misc.h > b/plugins/experimental/ats_speed/gzip/misc.h > new file mode 100644 > index 0000000..c44fb6b > --- /dev/null > +++ b/plugins/experimental/ats_speed/gzip/misc.h > @@ -0,0 +1,84 @@ > +/** @file > + > + Transforms content using gzip or deflate > + > + @section license License > + > + Licensed to the Apache Software Foundation (ASF) under one > + or more contributor license agreements. See the NOTICE file > + distributed with this work for additional information > + regarding copyright ownership. The ASF licenses this file > + to you under the Apache License, Version 2.0 (the > + "License"); you may not use this file except in compliance > + with the License. You may obtain a copy of the License at > + > + http://www.apache.org/licenses/LICENSE-2.0 > + > + Unless required by applicable law or agreed to in writing, software > + distributed under the License is distributed on an "AS IS" BASIS, > + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > + See the License for the specific language governing permissions and > + limitations under the License. > + */ > + > +#ifndef _GZIP_MISC_H_ > +#define _GZIP_MISC_H_ > + > +#define __STDC_LIMIT_MACROS > +#define __STDC_CONSTANT_MACROS > +#include <stdint.h> > +#include <zlib.h> > +#include <ts/ts.h> > +#include <stdlib.h> //exit() > +#include <stdio.h> > + > +//zlib stuff, see [deflateInit2] at http://www.zlib.net/manual.html > +static const int ZLIB_MEMLEVEL = 9; //min=1 (optimize for memory),max=9 > (optimized for speed) > +static const int WINDOW_BITS_DEFLATE = -15; > +static const int WINDOW_BITS_GZIP = 31; > + > +//misc > +static const int COMPRESSION_TYPE_DEFLATE = 1; > +static const int COMPRESSION_TYPE_GZIP = 2; > +//this one is just for txnargset/get to point to > +static const int GZIP_ONE = 1; > +static const int DICT_PATH_MAX = 512; > +static const int DICT_ENTRY_MAX = 2048; > + > +//this one is used to rename the accept encoding header > +//it will be restored later on > +//to make it work, the name must be different then downstream proxies though > +//otherwise the downstream will restore the accept encoding header > + > +enum transform_state > +{ > + transform_state_initialized, > + transform_state_output, > + transform_state_finished > +}; > + > +typedef struct > +{ > + TSHttpTxn txn; > + TSVIO downstream_vio; > + TSIOBuffer downstream_buffer; > + TSIOBufferReader downstream_reader; > + int downstream_length; > + z_stream zstrm; > + enum transform_state state; > + int compression_type; > +} GzipData; > + > + > +voidpf gzip_alloc(voidpf opaque, uInt items, uInt size); > +void gzip_free(voidpf opaque, voidpf address); > +void normalize_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc > hdr_loc); > +void hide_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, > const char * hidden_header_name); > +void restore_accept_encoding(TSHttpTxn txnp, TSMBuffer reqp, TSMLoc hdr_loc, > const char * hidden_header_name); > +const char * init_hidden_header_name(); > +int check_ts_version(); > +int register_plugin(); > +const char *load_dictionary(const char *preload_file); > +void gzip_log_ratio(int64_t in, int64_t out); > + > +#endif > > http://git-wip-us.apache.org/repos/asf/trafficserver/blob/083abd4f/plugins/experimental/ats_speed/scripts/prepare_psol.sh > ---------------------------------------------------------------------- > diff --git a/plugins/experimental/ats_speed/scripts/prepare_psol.sh > b/plugins/experimental/ats_speed/scripts/prepare_psol.sh > new file mode 100755 > index 0000000..4862249 > --- /dev/null > +++ b/plugins/experimental/ats_speed/scripts/prepare_psol.sh > @@ -0,0 +1,95 @@ > +#!/bin/bash > +# > +# Copyright 2012 Google Inc. > +# > +# Licensed under the Apache License, Version 2.0 (the "License"); > +# you may not use this file except in compliance with the License. > +# You may obtain a copy of the License at > +# > +# http://www.apache.org/licenses/LICENSE-2.0 > +# > +# Unless required by applicable law or agreed to in writing, software > +# distributed under the License is distributed on an "AS IS" BASIS, > +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. > +# See the License for the specific language governing permissions and > +# limitations under the License. > +# > +# Author: jef...@google.com (Jeff Kaufman) > +# Author: osch...@we-amp.com (Otto van der Schaaf) > +# > +# Usage: > +# scripts/prepare_psol.sh /path/to/mod_pagespeed/src > +# > +# Creates a directory psol/ and copies headers and a few source files from a > +# depot_tools (glient) checkout into psol/include. Along with creating > +# binaries, this is a step in preparing psol.tar.gz for distribution. > +# > + > +set -u # check for undefined variables > +set -e # exit on failed commands > + > +if [ "$(basename "$PWD")" != "ats_speed" ] ; then > + echo "$(basename $0) must be invoked from the ats_speed directory" > + exit 1 > +fi > + > +if [ $# -ne 1 ] ; then > + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src" > + exit 1 > +fi > + > +MOD_PAGESPEED_SRC="$1" > + > +if [ "$(basename "$(dirname "$MOD_PAGESPEED_SRC")")/$( \ > + basename "$MOD_PAGESPEED_SRC")" != "mod_pagespeed/src" ] ; then > + echo "Usage: $(basename $0) /path/to/mod_pagespeed/src" > + exit 1 > +fi > + > +if [ -e psol ] ; then > + echo "A psol/ directory already exists. Move it somewhere else and rerun." > + exit 1 > +fi > +mkdir psol/ > +# Copy over the .h files, plus a few selected .cc and .c files. > +rsync -arvz "$MOD_PAGESPEED_SRC/" "psol/include/" --prune-empty-dirs \ > + --exclude=".svn" \ > + --exclude=".git" \ > + --include='*.h' \ > + --include='*/' \ > + --include="apr_thread_compatible_pool.cc" \ > + --include="serf_url_async_fetcher.cc" \ > + --include="apr_mem_cache.cc" \ > + --include="key_value_codec.cc" \ > + --include="apr_memcache2.c" \ > + --include="loopback_route_fetcher.cc" \ > + --include="add_headers_fetcher.cc" \ > + --include="console_css_out.cc" \ > + --include="console_out.cc" \ > + --include="dense_hash_map" \ > + --include="dense_hash_set" \ > + --include="sparse_hash_map" \ > + --include="sparse_hash_set" \ > + --include="sparsetable" \ > + --include="mod_pagespeed_console_out.cc" \ > + --include="mod_pagespeed_console_css_out.cc" \ > + --include="mod_pagespeed_console_html_out.cc" \ > + --exclude='*' > +mkdir -p psol/lib/Debug/linux/ia32 > +mkdir -p psol/lib/Debug/linux/x64 > +mkdir -p psol/lib/Release/linux/ia32 > +mkdir -p psol/lib/Release/linux/x64 > + > +# Log that we did this. > +SVN_REVISION="$(svn info $MOD_PAGESPEED_SRC | grep Revision | awk '{print > $2}')" > +SVN_TAG="$(svn info $MOD_PAGESPEED_SRC | grep URL | awk -F/ '{print > $(NF-1)}')" > + > +DATE="$(date +%F)" > +echo "${DATE}: Copied from mod_pagespeed ${SVN_TAG}@r${SVN_REVISION} > ($USER)" \ > + >> psol/include_history.txt > + > +echo > +echo "Output is in psol/include. Now put binaries in psol/lib following" > +echo > "https://github.com/pagespeed/ngx_pagespeed/wiki/Building-Release-Binaries"; > +echo "and then you can distribute PSOL." > + >
