Hi Leif,
On the first range request it works in deed, the problem arises - at least
in my case - when the full file is cached by a previous request and the
range is served from the cache. I attached my heavily modified gzip (now
null-transform) plugin I used to test it.
Best Regards,
Sebastian
-----Ursprüngliche Nachricht-----
Von: Leif Hedstrom [mailto:zw...@apache.org]
Gesendet: Freitag, 9. August 2013 12:34
An: dev@trafficserver.apache.org
Betreff: Re: Plugin transforming between cache and end-user
On Aug 8, 2013, at 12:56 PM, Sebastian Annies
<sebastian.ann...@castlabs.com> wrote:
> Hi,
>
> So far I'm happy with everything except HTTP range requests.
> Whenever the plugin is null-transforming a HTTP range request the
> following is happening:
>
> * the return code is set to 200 (instead of 206)
> * the content-range header is omitted
> * nevertheless the plugin only 'sees' the requested bytes
Hmmm, I tested this with 3.3.5, and I'm not able to reproduce. I'm running
with the standard null-transform that comes in the examples/ directory.
For example:
loki (13:32) 424/0 $ curl -D - -o /dev/null -s -H "Range: bytes=0-100" -x
localhost:80
http://www.ogre.com/~leif/ats/Screenshot%202013-07-11%20at%2012.46.21%20PM
.png
HTTP/1.1 206 Partial Content
Date: Fri, 09 Aug 2013 19:32:20 GMT
Server: ATS/3.3.5-dev
Last-Modified: Thu, 11 Jul 2013 18:52:25 GMT
ETag: "51ab-4e140e52805cd"
Accept-Ranges: bytes
Content-Length: 101
Content-Range: bytes 0-100/20907
This seems exactly as expected (and I get the same result without the
null-transform.so plugin).
More details on how to reproduce this ? Are you serving out of ATS's
cache, or is it fetching from origin ? (My test fetches from Origin,
because the default null-transform.so doesn't apply to cache hits).
-- Leif
/** @file
Transforms content using gzip or deflate
@section license License
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#include <string>
#include <string.h>
#include <zlib.h>
#include <ts/ts.h>
#include "debug_macros.h"
#define INT64_MAX (9223372036854775807LL)
using namespace std;
int arg_idx_hooked;
int arg_idx_host_configuration;
int arg_idx_url_disallowed;
typedef struct
{
TSVIO output_vio;
TSIOBuffer output_buffer;
TSIOBufferReader output_reader;
} MyData;
static MyData *
my_data_alloc()
{
MyData *data;
data = (MyData *) TSmalloc(sizeof(MyData));
data->output_vio = NULL;
data->output_buffer = NULL;
data->output_reader = NULL;
return data;
}
static void
my_data_destroy(MyData * data)
{
if (data) {
if (data->output_buffer)
TSIOBufferDestroy(data->output_buffer);
TSfree(data);
}
}
const char *dictionary = NULL;
static void
gzip_transform_do(TSCont contp)
{
TSVConn output_conn;
TSIOBuffer buf_test;
TSVIO input_vio;
MyData *data;
int64_t towrite;
int64_t written;
int64_t avail;
debug("Entering handle_transform()");
/* Get the output (downstream) vconnection where we'll write data to. */
output_conn = TSTransformOutputVConnGet(contp);
/* Get the write VIO for the write operation that was performed on
* ourself. This VIO contains the buffer that we are to read from
* as well as the continuation we are to call when the buffer is
* empty. This is the input VIO (the write VIO for the upstream
* vconnection).
*/
input_vio = TSVConnWriteVIOGet(contp);
/* Get our data structure for this operation. The private data
* structure contains the output VIO and output buffer. If the
* private data structure pointer is NULL, then we'll create it
* and initialize its internals.
*/
data = (MyData *)TSContDataGet(contp);
if (!data) {
data = my_data_alloc();
data->output_buffer = TSIOBufferCreate();
data->output_reader = TSIOBufferReaderAlloc(data->output_buffer);
debug("Writing %ld bytes on VConn", TSVIONBytesGet(input_vio));
//data->output_vio = TSVConnWrite(output_conn, contp,
data->output_reader, INT32_MAX);
data->output_vio = TSVConnWrite(output_conn, contp, data->output_reader,
INT64_MAX);
// data->output_vio = TSVConnWrite(output_conn, contp,
data->output_reader, TSVIONBytesGet(input_vio));
TSContDataSet(contp, data);
}
/* We also check to see if the input VIO's buffer is non-NULL. A
* NULL buffer indicates that the write operation has been
* shutdown and that the upstream continuation does not want us to send any
* more WRITE_READY or WRITE_COMPLETE events. For this simplistic
* transformation that means we're done. In a more complex
* transformation we might have to finish writing the transformed
* data to our output connection.
*/
buf_test = TSVIOBufferGet(input_vio);
if (!buf_test) {
TSVIONBytesSet(data->output_vio, TSVIONDoneGet(input_vio));
TSVIOReenable(data->output_vio);
return;
}
/* Determine how much data we have left to read. For this null
* transform plugin this is also the amount of data we have left
* to write to the output connection.
*/
towrite = TSVIONTodoGet(input_vio);
written = TSVIONDoneGet(input_vio);
debug("toWrite is %ld", towrite);
debug("toWritten is %ld", written);
if (towrite > 0) {
/* The amount of data left to read needs to be truncated by
* the amount of data actually in the read buffer.
*/
avail = TSIOBufferReaderAvail(TSVIOReaderGet(input_vio));
debug("avail is %ld", avail);
if (towrite > avail) {
towrite = avail;
}
if (towrite > 0) {
/* Copy the data from the read buffer to the output buffer. */
TSIOBufferCopy(TSVIOBufferGet(data->output_vio),
TSVIOReaderGet(input_vio), towrite, 0);
/* Tell the read buffer that we have read the data and are no
* longer interested in it.
*/
TSIOBufferReaderConsume(TSVIOReaderGet(input_vio), towrite);
/* Modify the input VIO to reflect how much data we've
* completed.
*/
TSVIONDoneSet(input_vio, TSVIONDoneGet(input_vio) + towrite);
}
}
/* Now we check the input VIO to see if there is data left to
* read.
*/
if (TSVIONTodoGet(input_vio) > 0) {
if (towrite > 0) {
/* If there is data left to read, then we reenable the output
* connection by reenabling the output VIO. This will wake up
* the output connection and allow it to consume data from the
* output buffer.
*/
TSVIOReenable(data->output_vio);
/* Call back the input VIO continuation to let it know that we
* are ready for more data.
*/
TSContCall(TSVIOContGet(input_vio), TS_EVENT_VCONN_WRITE_READY,
input_vio);
}
} else {
/* If there is no data left to read, then we modify the output
* VIO to reflect how much data the output connection should
* expect. This allows the output connection to know when it
* is done reading. We then reenable the output connection so
* that it can consume the data we just gave it.
*/
TSVIONBytesSet(data->output_vio, TSVIONDoneGet(input_vio));
TSVIOReenable(data->output_vio);
/* Call back the input VIO continuation to let it know that we
* have completed the write operation.
*/
TSContCall(TSVIOContGet(input_vio), TS_EVENT_VCONN_WRITE_COMPLETE,
input_vio);
}
}
static int
gzip_transform(TSCont contp, TSEvent event, void * /* edata ATS_UNUSED */)
{
if (TSVConnClosedGet(contp)) {
my_data_destroy((MyData*)TSContDataGet(contp));
TSContDestroy(contp);
return 0;
} else {
switch (event) {
case TS_EVENT_ERROR:{
debug("gzip_transform: TS_EVENT_ERROR starts");
TSVIO upstream_vio = TSVConnWriteVIOGet(contp);
TSContCall(TSVIOContGet(upstream_vio), TS_EVENT_ERROR, upstream_vio);
}
break;
case TS_EVENT_VCONN_WRITE_COMPLETE:
TSVConnShutdown(TSTransformOutputVConnGet(contp), 0, 1);
break;
case TS_EVENT_VCONN_WRITE_READY:
gzip_transform_do(contp);
break;
case TS_EVENT_IMMEDIATE:
gzip_transform_do(contp);
break;
default:
warning("unknown event [%d]", event);
gzip_transform_do(contp);
break;
}
}
return 0;
}
static int
gzip_transformable(TSHttpTxn txnp, int server)
{
/* Server response header */
TSMBuffer bufp;
TSMLoc hdr_loc;
TSMLoc field_loc;
/* Client request header */
TSMBuffer cbuf;
TSMLoc chdr;
TSHttpStatus resp_status;
if (server) {
TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
} else {
TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
}
resp_status = TSHttpHdrStatusGet(bufp, hdr_loc);
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
//conservatively pick some statusses to compress
if (!(resp_status == 200 || resp_status == 206 || resp_status == 404 ||
resp_status == 500)) {
info("http response status [%d] is not compressible", resp_status);
return 0;
}
TSHttpTxnClientReqGet(txnp, &cbuf, &chdr);
//the only compressible method is currently GET.
int method_length;
const char *method = TSHttpHdrMethodGet(cbuf, chdr, &method_length);
if (!(method_length == TS_HTTP_LEN_GET && memcmp(method, TS_HTTP_METHOD_GET,
TS_HTTP_LEN_GET) == 0)) {
debug("method is not GET, not compressible");
TSHandleMLocRelease(cbuf, TS_NULL_MLOC, chdr);
return 0;
}
if (server) {
TSHttpTxnServerRespGet(txnp, &bufp, &hdr_loc);
} else {
TSHttpTxnCachedRespGet(txnp, &bufp, &hdr_loc);
}
/* If there already exists a content encoding then we don't want
to do anything. */
field_loc = TSMimeHdrFieldFind(bufp, hdr_loc, TS_MIME_FIELD_CONTENT_ENCODING,
-1);
if (field_loc) {
info("response is already content encoded, not compressible");
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
return 0;
}
TSHandleMLocRelease(bufp, hdr_loc, field_loc);
TSHandleMLocRelease(bufp, TS_NULL_MLOC, hdr_loc);
return 1;
}
static const int GZIP_ONE = 1;
static void
gzip_transform_add(TSHttpTxn txnp, int /* server ATS_UNUSED */)
{
int *tmp = (int *) TSHttpTxnArgGet(txnp, arg_idx_hooked);
if (tmp) {
//happens on cache_stale_hit
debug("transform hook already set, bail");
return;
} else {
TSHttpTxnArgSet(txnp, arg_idx_hooked, (void *) &GZIP_ONE);
info("adding compression transform");
}
TSHttpTxnUntransformedRespCache(txnp, 1);
// never cache tranformed
TSHttpTxnTransformedRespCache(txnp, 0);
TSVConn connp;
connp = TSTransformCreate(gzip_transform, txnp);
TSHttpTxnHookAdd(txnp, TS_HTTP_RESPONSE_TRANSFORM_HOOK, connp);
}
static int
cache_transformable(TSHttpTxn txnp)
{
int obj_status;
if (TSHttpTxnCacheLookupStatusGet(txnp, &obj_status) == TS_ERROR) {
warning("Couldn't get cache status of object");
return 0;
}
if (obj_status == TS_CACHE_LOOKUP_HIT_STALE) {
info("stale cache hit");
return 0;
}
if (obj_status == TS_CACHE_LOOKUP_HIT_FRESH) {
info("fresh cache hit");
return 1;
}
return 0;
}
static int
transform_plugin(TSCont /* contp ATS_UNUSED */, TSEvent event, void *edata)
{
TSHttpTxn txnp = (TSHttpTxn) edata;
switch (event) {
case TS_EVENT_HTTP_PRE_REMAP:
{
info("Pre-remap. Extract params here and put them into txn");
TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
}
break;
case TS_EVENT_HTTP_READ_REQUEST_HDR:
{
TSMBuffer req_buf;
TSMLoc req_loc;
if (TSHttpTxnClientReqGet(txnp, &req_buf, &req_loc) == TS_SUCCESS) {
int url_len;
char * url = TSHttpTxnEffectiveUrlStringGet(txnp, &url_len);
TSfree(url);
TSHandleMLocRelease(req_buf, TS_NULL_MLOC, req_loc);
}
TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
}
break;
case TS_EVENT_HTTP_READ_RESPONSE_HDR:
{
//os: the accept encoding header needs to be restored..
//otherwise the next request won't get a cache hit on this
if ( gzip_transformable(txnp, 1)) {
gzip_transform_add(txnp, 1);
}
TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
}
break;
case TS_EVENT_HTTP_CACHE_LOOKUP_COMPLETE:
{
if (cache_transformable(txnp) && gzip_transformable(txnp, 0)) {
gzip_transform_add(txnp, 0);
}
TSHttpTxnReenable(txnp, TS_EVENT_HTTP_CONTINUE);
}
break;
default:
fatal("gzip transform unknown event");
}
return 0;
}
void
TSPluginInit(int argc, const char *argv[])
{
string config_path;
if (argc > 2) {
fatal("the gzip plugin does not accept more than 1 plugin argument");
} else if (argc == 2) {
config_path = std::string(argv[1]);
}
info("TSPluginInit %s", argv[0]);
TSPluginRegistrationInfo info;
info.plugin_name = (char*)"gzip";
info.vendor_name = (char*)"Apache";
info.support_email = (char*)"dev@trafficserver.apache.org";
if (TSPluginRegister(TS_SDK_VERSION_3_0, &info) != TS_SUCCESS) {
fatal("The gzip plugin failed to register");
}
const char *ts_version = TSTrafficServerVersionGet();
TSReleaseAssert(ts_version);
int scan_result;
int major_version;
scan_result = sscanf(ts_version, "%d", &major_version);
TSReleaseAssert(scan_result == 1);
if (major_version < 3) {
fatal("The gzip plugin requires at least traffic server v3");
}
if (TSHttpArgIndexReserve("gzip", "for remembering if the hook was set",
&arg_idx_hooked) != TS_SUCCESS) {
fatal("failed to reserve an argument index");
}
if (TSHttpArgIndexReserve("gzip", "for storing if compression is applicable",
&arg_idx_host_configuration) != TS_SUCCESS) {
fatal("failed to reserve an argument index");
}
if (TSHttpArgIndexReserve("gzip", "for storing if compression is disallowed
for this txn", &arg_idx_url_disallowed) != TS_SUCCESS) {
fatal("failed to reserve an argument index");
}
//fixme: never freed. there is no shutdown event?
char * p = (char*)TSmalloc(config_path.size()+1);
strcpy(p,config_path.c_str());
TSCont transform_contp = TSContCreate(transform_plugin, NULL);
TSHttpHookAdd(TS_HTTP_READ_REQUEST_HDR_HOOK, transform_contp);
TSHttpHookAdd(TS_HTTP_READ_RESPONSE_HDR_HOOK, transform_contp);
TSHttpHookAdd(TS_HTTP_PRE_REMAP_HOOK, transform_contp);
TSHttpHookAdd(TS_HTTP_CACHE_LOOKUP_COMPLETE_HOOK, transform_contp);
info("loaded");
}
/** @file
Transforms content using gzip or deflate
@section license License
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
distributed with this work for additional information
regarding copyright ownership. The ASF licenses this file
to you under the Apache License, Version 2.0 (the
"License"); you may not use this file except in compliance
with the License. You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
#ifndef _DBG_MACROS_H
#define _DBG_MACROS_H
#include <ts/ts.h>
#include <stdlib.h>
#include <stdio.h>
#define TAG "gzip"
#define debug(fmt, args...) do { \
TSDebug(TAG, "DEBUG: [%s:%d] [%s] " fmt, __FILE__, __LINE__, __FUNCTION__ ,
##args ); \
} while (0)
#define info(fmt, args...) do { \
TSDebug(TAG, "INFO: " fmt, ##args ); \
} while (0)
#define warning(fmt, args...) do { \
TSDebug(TAG, "WARNING: " fmt, ##args ); \
} while (0)
#define error(fmt, args...) do { \
TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args
); \
TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ ,
##args ); \
} while (0)
#define fatal(fmt, args...) do { \
TSError("[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ , ##args
); \
TSDebug(TAG, "[%s:%d] [%s] ERROR: " fmt, __FILE__, __LINE__, __FUNCTION__ ,
##args ); \
exit(-1); \
} while (0)
//FIXME: this one doesn't deserve to be here
#define DISALLOW_COPY_AND_ASSIGN(TypeName) \
TypeName(const TypeName&); \
void operator=(const TypeName&)
#endif //_DBG_MACROS_H
