bes  Updated for version 3.20.10
SuperChunk.h
1 // -*- mode: c++; c-basic-offset:4 -*-
2 
3 // This file is part of the BES
4 
5 // Copyright (c) 2020 OPeNDAP, Inc.
6 // Author: Nathan Potter<ndp@opendap.org>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
23 
24 #ifndef HYRAX_GIT_SUPERCHUNK_H
25 #define HYRAX_GIT_SUPERCHUNK_H
26 
27 
28 #include <vector>
29 #include <memory>
30 #include <thread>
31 #include <queue>
32 #include <sstream>
33 
34 
35 #include "Chunk.h"
36 
37 
38 namespace dmrpp {
39 
40 // Forward Declaration;
41 class DmrppArray;
42 
47 class SuperChunk {
48 private:
49  std::string d_id;
50  DmrppArray *d_parent_array;
51  std::shared_ptr<http::url> d_data_url;
52  std::vector<std::shared_ptr<Chunk>> d_chunks;
53  unsigned long long d_offset;
54  unsigned long long d_size;
55  bool d_is_read;
56  char *d_read_buffer;
57 
58  bool is_contiguous(std::shared_ptr<Chunk> candidate_chunk);
59  void map_chunks_to_buffer();
60  void read_aggregate_bytes();
61 
62 public:
63 
64  explicit SuperChunk(const std::string sc_id, DmrppArray *parent=nullptr):
65  d_id(sc_id), d_parent_array(parent), d_data_url(nullptr), d_offset(0), d_size(0), d_is_read(false), d_read_buffer(nullptr){}
66 
67  virtual ~SuperChunk(){
68  delete[] d_read_buffer;
69  }
70 
71  virtual std::string id(){ return d_id; }
72 
73  virtual bool add_chunk(std::shared_ptr<Chunk> candidate_chunk);
74 
75  std::shared_ptr<http::url> get_data_url(){ return d_data_url; }
76  virtual unsigned long long get_size(){ return d_size; }
77  virtual unsigned long long get_offset(){ return d_offset; }
78 
79  virtual void read(){
80  retrieve_data();
82  }
83  virtual void read_unconstrained(){
84  retrieve_data();
86  }
87 
88  virtual void retrieve_data();
89  virtual void process_child_chunks();
91 
92 
93  virtual bool empty(){ return d_chunks.empty(); }
94 
95  std::vector<std::shared_ptr<Chunk>> get_chunks(){ return d_chunks; }
96 
97  std::string to_string(bool verbose) const;
98  virtual void dump(std::ostream & strm) const;
99 };
100 
106  std::thread::id parent_thread_id;
107  std::string parent_super_chunk_id;
108  std::shared_ptr<Chunk> chunk;
109  DmrppArray *array;
110  const vector<unsigned long long> &array_shape;
111 
112  one_chunk_args(const string sc_id, std::shared_ptr<Chunk> c, DmrppArray *a, const vector<unsigned long long> &a_s)
113  : parent_thread_id(std::this_thread::get_id()), parent_super_chunk_id(sc_id), chunk(std::move(c)), array(a), array_shape(a_s) {}
114 };
115 
122  std::thread::id parent_thread_id;
123  std::string parent_super_chunk_id;
124  std::shared_ptr<Chunk> chunk;
125  DmrppArray *array;
126  const vector<unsigned long long> &array_shape;
127  const vector<unsigned long long> &chunk_shape;
128 
129  one_chunk_unconstrained_args(const string sc_id, std::shared_ptr<Chunk> c, DmrppArray *a, const vector<unsigned long long> &a_s,
130  const vector<unsigned long long> &c_s)
131  : parent_thread_id(std::this_thread::get_id()), parent_super_chunk_id(sc_id), chunk(std::move(c)),
132  array(a), array_shape(a_s), chunk_shape(c_s) {}
133 };
134 
135 void process_chunks_concurrent(
136  const string &super_chunk_id,
137  std::queue<shared_ptr<Chunk>> &chunks,
138  DmrppArray *array,
139  const std::vector<unsigned long long> &shape );
140 
141 void process_chunks_unconstrained_concurrent(
142  const string &super_chunk_id,
143  std::queue<std::shared_ptr<Chunk>> &chunks,
144  const std::vector<unsigned long long> &chunk_shape,
145  DmrppArray *array,
146  const std::vector<unsigned long long> &array_shape);
147 
148 
149 
150 }// namespace dmrpp
151 
152 
153 #endif //HYRAX_GIT_SUPERCHUNK_H
Extend libdap::Array so that a handler can read data using a DMR++ file.
Definition: DmrppArray.h:68
A SuperChunk is a collection of contiguous Chunk objects along with optimized methods for data retrie...
Definition: SuperChunk.h:47
virtual void retrieve_data()
Cause the SuperChunk and all of it's subordinate Chunks to be read.
Definition: SuperChunk.cc:503
virtual bool add_chunk(std::shared_ptr< Chunk > candidate_chunk)
Attempts to add a new Chunk to this SuperChunk.
Definition: SuperChunk.cc:400
std::string to_string(bool verbose) const
Makes a string representation of the SuperChunk.
Definition: SuperChunk.cc:619
virtual void dump(std::ostream &strm) const
Writes the to_string() output to the stream strm.
Definition: SuperChunk.cc:640
virtual void process_child_chunks()
Reads the SuperChunk, inflates/deshuffles the subordinate chunks as required and copies the values in...
Definition: SuperChunk.cc:540
virtual void process_child_chunks_unconstrained()
Reads the SuperChunk, inflates/deshuffles the subordinate chunks as required and copies the values in...
Definition: SuperChunk.cc:578
Single argument structure for a thread that will process a single Chunk for a constrained array....
Definition: SuperChunk.h:105
Single argument structure for a thread that will process a single Chunk for an unconstrained array....
Definition: SuperChunk.h:121