36#include <libdap/util.h>
42#define S3_TRACKING_CONTEXT "cloudydap"
43#define ENABLE_TRACKING_QUERY_PARAMETER 0
48size_t chunk_header_callback(
char *buffer,
size_t size,
size_t nitems,
void *data);
49size_t chunk_write_data(
void *buffer,
size_t size,
size_t nmemb,
void *data);
51void process_s3_error_response(
const std::shared_ptr<http::url> &data_url,
const std::string &xml_message);
61 std::shared_ptr<http::url> d_data_url;
62 std::string d_query_marker;
63 std::string d_byte_order;
64 unsigned long long d_size;
65 unsigned long long d_offset;
67 std::vector<unsigned long long> d_chunk_position_in_array;
89 bool d_read_buffer_is_mine;
90 unsigned long long d_bytes_read;
92 unsigned long long d_read_buffer_size;
95 std::string d_response_content_type;
100 friend class ChunkTest;
101 friend class DmrppCommonTest;
102 friend class MockChunk;
106 void _duplicate(
const Chunk &bs)
109 d_read_buffer_is_mine =
true;
111 d_read_buffer =
nullptr;
112 d_read_buffer_size = 0;
114 d_is_inflated =
false;
117 d_offset = bs.d_offset;
118 d_data_url = bs.d_data_url;
119 d_byte_order = bs.d_byte_order;
120 d_query_marker = bs.d_query_marker;
121 d_chunk_position_in_array = bs.d_chunk_position_in_array;
136 d_data_url(nullptr), d_size(0), d_offset(0),
137 d_read_buffer_is_mine(true), d_bytes_read(0), d_read_buffer(nullptr),
138 d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
153 std::shared_ptr<http::url> data_url,
155 unsigned long long size,
156 unsigned long long offset,
157 const std::string &pia_str =
"") :
158 d_data_url(std::move(data_url)),
159 d_byte_order(std::move(order)),
162 d_read_buffer_is_mine(true),
164 d_read_buffer(nullptr),
165 d_read_buffer_size(0),
169#if ENABLE_TRACKING_QUERY_PARAMETER
187 unsigned long long size,
188 unsigned long long offset,
189 const std::string &pia_str =
"") :
190 d_byte_order(std::move(order)),
193 d_read_buffer_is_mine(true),
195 d_read_buffer(nullptr),
196 d_read_buffer_size(0),
200#if ENABLE_TRACKING_QUERY_PARAMETER
217 std::shared_ptr<http::url> data_url,
219 unsigned long long size,
220 unsigned long long offset,
221 const std::vector<unsigned long long> &pia_vec) :
222 d_data_url(std::move(data_url)),
223 d_byte_order(std::move(order)),
226 d_read_buffer_is_mine(true),
228 d_read_buffer(nullptr),
229 d_read_buffer_size(0),
233#if ENABLE_TRACKING_QUERY_PARAMETER
252 unsigned long long size,
253 unsigned long long offset,
254 const std::vector<unsigned long long> &pia_vec) :
256 d_byte_order(std::move(order)),
257 d_size(size), d_offset(offset),
258 d_read_buffer_is_mine(true),
260 d_read_buffer(nullptr),
261 d_read_buffer_size(0),
265#if ENABLE_TRACKING_QUERY_PARAMETER
279 if(d_read_buffer_is_mine)
280 delete[] d_read_buffer;
281 d_read_buffer =
nullptr;
290 if (
this == &rhs)
return *
this;
325 virtual std::shared_ptr<http::url>
get_data_url()
const;
332 d_data_url = std::move(data_url);
349 d_bytes_read = bytes_read;
368 if(d_read_buffer_is_mine)
369 delete[] d_read_buffer;
371 d_read_buffer =
new char[d_size];
372 d_read_buffer_size = d_size;
373 d_read_buffer_is_mine =
true;
383 return d_read_buffer;
400 virtual void set_rbuf(char *buf, unsigned int buf_size)
402 if(d_read_buffer_is_mine)
403 delete[] d_read_buffer;
406 d_read_buffer_size = buf_size;
425 unsigned long long buf_size,
426 unsigned long long bytes_read = 0,
427 bool assume_ownership =
true ){
429 if(d_read_buffer_is_mine)
430 delete[] d_read_buffer;
432 d_read_buffer_is_mine = assume_ownership;
434 d_read_buffer_size = buf_size;
444 return d_read_buffer_size;
452 return d_chunk_position_in_array;
463 virtual void filter_chunk(
const std::string &filters,
unsigned long long chunk_size,
unsigned long long elem_width);
465 virtual bool get_is_read() {
return d_is_read; }
466 virtual void set_is_read(
bool state) { d_is_read = state; }
473 static void parse_chunk_position_in_array_string(
const std::string &pia, std::vector<unsigned long long> &pia_vect);
475 virtual void dump(std::ostream & strm)
const;
477 virtual std::string to_string()
const;
482struct inflate_chunk_args {
486 unsigned int chunk_size;
487 unsigned int elem_width;
489 inflate_chunk_args(Chunk *c,
bool d,
bool s,
unsigned int c_size,
unsigned int e_size):
490 chunk(c), deflate(d), shuffle(s), chunk_size(c_size), elem_width(e_size) {}
493void *inflate_chunk(
void *args);
virtual void set_bytes_read(unsigned long long bytes_read)
Set the size of this Chunk's data block.
virtual void dump(std::ostream &strm) const
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values, the data URL will not be set.
virtual char * get_rbuf()
virtual void read_chunk()
void add_tracking_query_param()
Modify this chunk's data URL so that it includes tracking info.
virtual std::string get_response_content_type()
Get the response type of the last response.
virtual std::string get_byte_order()
Get the chunk byte order.
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
virtual std::shared_ptr< http::url > get_data_url() const
Get the data url for this Chunk's data block.
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values.
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values, the data URl will not be set.
virtual void set_data_url(std::shared_ptr< http::url > data_url)
Set the data url for this Chunk's data block.
Chunk()
Get an empty chunk.
Chunk & operator=(const Chunk &rhs)
virtual void set_rbuf_to_size()
Allocates the internal read buffer to be d_size bytes.
virtual unsigned long long get_offset() const
Get the offset to this Chunk's data block.
virtual unsigned long long get_bytes_read() const
Get the number of bytes read so far for this Chunk.
void set_position_in_array(const std::string &pia)
parse the chunk position string
virtual unsigned long long get_rbuf_size() const
virtual unsigned long long get_size() const
Get the size of this Chunk's data block on disk.
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values.
virtual const std::vector< unsigned long long > & get_position_in_array() const
void set_read_buffer(char *buf, unsigned long long buf_size, unsigned long long bytes_read=0, bool assume_ownership=true)
Set the target read buffer for this chunk.
virtual void filter_chunk(const std::string &filters, unsigned long long chunk_size, unsigned long long elem_width)
filter data in the chunk
void set_response_content_type(const std::string &ct)
Set the response type of the last response.