bes  Updated for version 3.20.10
Chunk.h
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of the BES
4 
5 // Copyright (c) 2016 OPeNDAP, Inc.
6 // Author: Nathan Potter <ndp@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 
24 #ifndef _Chunk_h
25 #define _Chunk_h 1
26 
27 #include <string>
28 #include <utility>
29 #include <vector>
30 #include <memory>
31 
32 // BES
33 #include "url_impl.h"
34 
35 // libdap4
36 #include <libdap/util.h>
37 
38 
39 // This is used to track access to 'cloudydap' accesses in the S3 logs
40 // by adding a query string that will show up in those logs. This is
41 // activated by using a special BES context with the name 'cloudydap.'
42 #define S3_TRACKING_CONTEXT "cloudydap"
43 #define ENABLE_TRACKING_QUERY_PARAMETER 0
44 
45 namespace dmrpp {
46 
47 // Callback functions used by chunk readers
48 size_t chunk_header_callback(char *buffer, size_t size, size_t nitems, void *data);
49 size_t chunk_write_data(void *buffer, size_t size, size_t nmemb, void *data);
50 
51 void process_s3_error_response(const std::shared_ptr<http::url> &data_url, const std::string &xml_message);
52 
59 class Chunk {
60 private:
61  std::shared_ptr<http::url> d_data_url;
62  std::string d_query_marker;
63  std::string d_byte_order;
64  unsigned long long d_size;
65  unsigned long long d_offset;
66 
67  std::vector<unsigned long long> d_chunk_position_in_array;
68 
69  // These are used only during the libcurl callback;
70  // they are not duplicated by the copy ctor or assignment
71  // operator.
72 
89  bool d_read_buffer_is_mine;
90  unsigned long long d_bytes_read;
91  char *d_read_buffer;
92  unsigned long long d_read_buffer_size;
93  bool d_is_read;
94  bool d_is_inflated;
95  std::string d_response_content_type;
96 
97  // static const std::string tracking_context;
98 
99 
100  friend class ChunkTest;
101  friend class DmrppCommonTest;
102  friend class MockChunk;
103 
104 protected:
105 
106  void _duplicate(const Chunk &bs)
107  {
108  // See above
109  d_read_buffer_is_mine = true;
110  d_bytes_read = 0;
111  d_read_buffer = nullptr;
112  d_read_buffer_size = 0;
113  d_is_read = false;
114  d_is_inflated = false;
115 
116  d_size = bs.d_size;
117  d_offset = bs.d_offset;
118  d_data_url = bs.d_data_url;
119  d_byte_order = bs.d_byte_order;
120  d_query_marker = bs.d_query_marker;
121  d_chunk_position_in_array = bs.d_chunk_position_in_array;
122  }
123 
124 public:
125 
135  Chunk() :
136  d_data_url(nullptr), d_size(0), d_offset(0),
137  d_read_buffer_is_mine(true), d_bytes_read(0), d_read_buffer(nullptr),
138  d_read_buffer_size(0), d_is_read(false), d_is_inflated(false)
139  {
140  }
141 
153  std::shared_ptr<http::url> data_url,
154  std::string order,
155  unsigned long long size,
156  unsigned long long offset,
157  const std::string &pia_str = "") :
158  d_data_url(std::move(data_url)),
159  d_byte_order(std::move(order)),
160  d_size(size),
161  d_offset(offset),
162  d_read_buffer_is_mine(true),
163  d_bytes_read(0),
164  d_read_buffer(nullptr),
165  d_read_buffer_size(0),
166  d_is_read(false),
167  d_is_inflated(false)
168  {
169 #if ENABLE_TRACKING_QUERY_PARAMETER
171 #endif
172  set_position_in_array(pia_str);
173  }
174 
186  std::string order,
187  unsigned long long size,
188  unsigned long long offset,
189  const std::string &pia_str = "") :
190  d_byte_order(std::move(order)),
191  d_size(size),
192  d_offset(offset),
193  d_read_buffer_is_mine(true),
194  d_bytes_read(0),
195  d_read_buffer(nullptr),
196  d_read_buffer_size(0),
197  d_is_read(false),
198  d_is_inflated(false)
199  {
200 #if ENABLE_TRACKING_QUERY_PARAMETER
202 #endif
203  set_position_in_array(pia_str);
204  }
205 
217  std::shared_ptr<http::url> data_url,
218  std::string order,
219  unsigned long long size,
220  unsigned long long offset,
221  const std::vector<unsigned long long> &pia_vec) :
222  d_data_url(std::move(data_url)),
223  d_byte_order(std::move(order)),
224  d_size(size),
225  d_offset(offset),
226  d_read_buffer_is_mine(true),
227  d_bytes_read(0),
228  d_read_buffer(nullptr),
229  d_read_buffer_size(0),
230  d_is_read(false),
231  d_is_inflated(false)
232  {
233 #if ENABLE_TRACKING_QUERY_PARAMETER
235 #endif
236  set_position_in_array(pia_vec);
237  }
238 
239 
251  std::string order,
252  unsigned long long size,
253  unsigned long long offset,
254  const std::vector<unsigned long long> &pia_vec) :
255  d_query_marker(""),
256  d_byte_order(std::move(order)),
257  d_size(size), d_offset(offset),
258  d_read_buffer_is_mine(true),
259  d_bytes_read(0),
260  d_read_buffer(nullptr),
261  d_read_buffer_size(0),
262  d_is_read(false),
263  d_is_inflated(false)
264  {
265 #if ENABLE_TRACKING_QUERY_PARAMETER
267 #endif
268  set_position_in_array(pia_vec);
269  }
270 
271 
272  Chunk(const Chunk &h4bs)
273  {
274  _duplicate(h4bs);
275  }
276 
277  virtual ~Chunk()
278  {
279  if(d_read_buffer_is_mine)
280  delete[] d_read_buffer;
281  d_read_buffer = nullptr;
282  }
283 
288  Chunk &operator=(const Chunk &rhs)
289  {
290  if (this == &rhs) return *this;
291 
292  _duplicate(rhs);
293 
294  return *this;
295  }
296 
298  virtual std::string get_response_content_type() { return d_response_content_type; }
299 
301  void set_response_content_type(const std::string &ct) { d_response_content_type = ct; }
302 
304  virtual std::string get_byte_order() { return d_byte_order; }
305 
309  virtual unsigned long long get_size() const
310  {
311  return d_size;
312  }
313 
317  virtual unsigned long long get_offset() const
318  {
319  return d_offset;
320  }
321 
325  virtual std::shared_ptr<http::url> get_data_url() const;
326 
330  virtual void set_data_url(std::shared_ptr<http::url> data_url)
331  {
332  d_data_url = std::move(data_url);
333  }
334 
338  virtual unsigned long long get_bytes_read() const
339  {
340  return d_bytes_read;
341  }
342 
347  virtual void set_bytes_read(unsigned long long bytes_read)
348  {
349  d_bytes_read = bytes_read;
350  }
351 
366  virtual void set_rbuf_to_size()
367  {
368  if(d_read_buffer_is_mine)
369  delete[] d_read_buffer;
370 
371  d_read_buffer = new char[d_size];
372  d_read_buffer_size = d_size;
373  d_read_buffer_is_mine = true;
374  set_bytes_read(0);
375  }
376 
381  virtual char *get_rbuf()
382  {
383  return d_read_buffer;
384  }
385 
386 #if 0 // Superseded by Chunk::set_read_buffer(); - ndp 12/17/20
400  virtual void set_rbuf(char *buf, unsigned int buf_size)
401  {
402  if(d_read_buffer_is_mine)
403  delete[] d_read_buffer;
404 
405  d_read_buffer = buf;
406  d_read_buffer_size = buf_size;
407 
408  set_bytes_read(buf_size);
409  }
410 #endif
411 
424  char *buf,
425  unsigned long long buf_size,
426  unsigned long long bytes_read = 0,
427  bool assume_ownership = true ){
428 
429  if(d_read_buffer_is_mine)
430  delete[] d_read_buffer;
431 
432  d_read_buffer_is_mine = assume_ownership;
433  d_read_buffer = buf;
434  d_read_buffer_size = buf_size;
435 
436  set_bytes_read(bytes_read);
437  }
438 
442  virtual unsigned long long get_rbuf_size() const
443  {
444  return d_read_buffer_size;
445  }
446 
450  virtual const std::vector<unsigned long long> &get_position_in_array() const
451  {
452  return d_chunk_position_in_array;
453  }
454 
455 
457 
458  void set_position_in_array(const std::string &pia);
459  void set_position_in_array(const std::vector<unsigned long long> &pia);
460 
461  virtual void read_chunk();
462 
463  virtual void filter_chunk(const std::string &filters, unsigned long long chunk_size, unsigned long long elem_width);
464 
465  virtual bool get_is_read() { return d_is_read; }
466  virtual void set_is_read(bool state) { d_is_read = state; }
467 
468  //virtual bool get_is_inflated() const { return d_is_inflated; }
469  //virtual void set_is_inflated(bool state) { d_is_inflated = state; }
470 
471  virtual std::string get_curl_range_arg_string();
472 
473  static void parse_chunk_position_in_array_string(const std::string &pia, std::vector<unsigned long long> &pia_vect);
474 
475  virtual void dump(std::ostream & strm) const;
476 
477  virtual std::string to_string() const;
478 };
479 
480 #if 0
482 struct inflate_chunk_args {
483  Chunk *chunk;
484  bool deflate;
485  bool shuffle;
486  unsigned int chunk_size;
487  unsigned int elem_width;
488 
489  inflate_chunk_args(Chunk *c, bool d, bool s, unsigned int c_size, unsigned int e_size):
490  chunk(c), deflate(d), shuffle(s), chunk_size(c_size), elem_width(e_size) {}
491 };
492 
493 void *inflate_chunk(void *args);
494 #endif
495 
496 } // namespace dmrpp
497 
498 #endif // _Chunk_h
virtual void set_bytes_read(unsigned long long bytes_read)
Set the size of this Chunk's data block.
Definition: Chunk.h:347
virtual void dump(std::ostream &strm) const
Definition: Chunk.cc:880
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values, the data URL will not be set.
Definition: Chunk.h:185
virtual void read_chunk()
Definition: Chunk.cc:840
void add_tracking_query_param()
Modify this chunk's data URL so that it includes tracking info.
Definition: Chunk.cc:526
virtual std::string get_response_content_type()
Get the response type of the last response.
Definition: Chunk.h:298
virtual const std::vector< unsigned long long > & get_position_in_array() const
Definition: Chunk.h:450
virtual std::string get_byte_order()
Get the chunk byte order.
Definition: Chunk.h:304
virtual std::string get_curl_range_arg_string()
Returns a curl range argument. The libcurl requires a string argument for range-ge activitys,...
Definition: Chunk.cc:509
virtual std::shared_ptr< http::url > get_data_url() const
Get the data url for this Chunk's data block.
Definition: Chunk.cc:903
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::string &pia_str="")
Get a chunk initialized with values.
Definition: Chunk.h:152
Chunk & operator=(const Chunk &rhs)
Definition: Chunk.h:288
Chunk(std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values, the data URl will not be set.
Definition: Chunk.h:250
virtual void set_data_url(std::shared_ptr< http::url > data_url)
Set the data url for this Chunk's data block.
Definition: Chunk.h:330
Chunk()
Get an empty chunk.
Definition: Chunk.h:135
virtual void set_rbuf_to_size()
Allocates the internal read buffer to be d_size bytes.
Definition: Chunk.h:366
virtual unsigned long long get_offset() const
Get the offset to this Chunk's data block.
Definition: Chunk.h:317
virtual unsigned long long get_bytes_read() const
Get the number of bytes read so far for this Chunk.
Definition: Chunk.h:338
void set_position_in_array(const std::string &pia)
parse the chunk position string
Definition: Chunk.cc:458
virtual unsigned long long get_rbuf_size() const
Definition: Chunk.h:442
virtual unsigned long long get_size() const
Get the size of this Chunk's data block on disk.
Definition: Chunk.h:309
Chunk(std::shared_ptr< http::url > data_url, std::string order, unsigned long long size, unsigned long long offset, const std::vector< unsigned long long > &pia_vec)
Get a chunk initialized with values.
Definition: Chunk.h:216
virtual char * get_rbuf()
Definition: Chunk.h:381
void set_read_buffer(char *buf, unsigned long long buf_size, unsigned long long bytes_read=0, bool assume_ownership=true)
Set the target read buffer for this chunk.
Definition: Chunk.h:423
virtual void filter_chunk(const std::string &filters, unsigned long long chunk_size, unsigned long long elem_width)
filter data in the chunk
Definition: Chunk.cc:755
void set_response_content_type(const std::string &ct)
Set the response type of the last response.
Definition: Chunk.h:301