34 #include <libdap/BaseType.h>
35 #include <libdap/D4Group.h>
37 #include "BESInternalError.h"
38 #include "BESInternalFatalError.h"
41 #include "TheBESKeys.h"
43 #include "BESStopWatch.h"
44 #include "BESIndent.h"
46 #include "DmrppNames.h"
48 #include "DmrppD4Group.h"
49 #include "DmrppArray.h"
50 #include "DmrppParserSax2.h"
51 #include "DmrppTypeFactory.h"
53 #include "SuperChunk.h"
55 #define prolog std::string("superchunky::").append(__func__).append("() - ")
61 void compute_super_chunks(
dmrpp::DmrppArray *array,
bool , vector<SuperChunk *> &super_chunks){
68 unsigned long long sc_count=0;
70 sc_id << array->name() <<
"-" << sc_count++;
73 auto currentSuperChunk =
new SuperChunk(sc_id.str(), array);
74 super_chunks.push_back(currentSuperChunk);
75 if(debug) cout <<
"SuperChunking array: "<< array->name() << endl;
77 for(
const auto &chunk:chunks){
78 bool was_added = currentSuperChunk->
add_chunk(chunk);
81 unsigned long long next_contiguous_chunk_offset = currentSuperChunk->get_offset() + currentSuperChunk->get_size();
82 unsigned long long gap_size;
83 bool is_behind =
false;
84 if(chunk->get_offset() > next_contiguous_chunk_offset){
85 gap_size = chunk->get_offset() - next_contiguous_chunk_offset;
89 gap_size = next_contiguous_chunk_offset - chunk->get_offset();
92 msg <<
"FOUND GAP chunk(offset: " << chunk->get_offset();
93 msg <<
" size: " << chunk->get_size() <<
")";
94 msg <<
" SuperChunk(ptr: " << (
void *) currentSuperChunk;
95 msg <<
" offset: " << currentSuperChunk->get_offset();
96 msg <<
" size: " << currentSuperChunk->get_size();
97 msg <<
" next_contiguous_chunk_offset: " << next_contiguous_chunk_offset <<
") ";
98 msg <<
" gap_size: " << gap_size;
99 msg <<
" bytes" << (is_behind?
" behind":
" beyond") <<
" target offset";
105 if(!currentSuperChunk->empty()){
106 sc_id.str(std::string());
107 sc_id << array->name() <<
"-" << sc_count++;
108 currentSuperChunk =
new SuperChunk(sc_id.str(), array);
109 super_chunks.push_back(currentSuperChunk);
111 bool add_first_successful = currentSuperChunk->add_chunk(chunk);
112 if(!add_first_successful)
113 throw BESInternalError(
"ERROR: Failed to add first Chunk to a new SuperChunk."+
114 chunk->to_string() ,__FILE__,__LINE__);
119 if(currentSuperChunk->empty()) {
120 super_chunks.pop_back();
121 delete currentSuperChunk;
124 cout <<
"SuperChunk Inventory For Array: " << array->name() << endl;
125 for(
auto super_chunk: super_chunks) {
126 cout << super_chunk->to_string(
true) << endl;
131 void compute_super_chunks(libdap::BaseType *var,
bool only_constrained, vector<SuperChunk *> &super_chunks) {
132 if (var->is_simple_type())
134 if (var->is_constructor_type())
136 if (var->is_vector_type()) {
139 if(debug) cout <<
"Found DmrppArray: "<< array->name() << endl;
140 compute_super_chunks(array, only_constrained, super_chunks);
143 BESDEBUG(MODULE, prolog <<
"The variable: "<< var->name()
144 <<
" is not an instance of DmrppArray. SKIPPING"<< endl);
150 void inventory_super_chunks(libdap::BaseType *var,
bool only_constrained, vector<SuperChunk *> &super_chunks){
151 if(var->is_simple_type())
153 if(var->is_constructor_type())
155 if(var->is_vector_type()){
156 auto array =
dynamic_cast<DmrppArray*
>(var);
162 unsigned long long next_contiguous_chunk_offset = 0;
165 vector<vector<const Chunk *> *> super_chunks;
166 auto currentSuperChunk =
new vector<const Chunk *>();
167 super_chunks.push_back(currentSuperChunk);
169 if(debug) cout <<
"SuperChunking array: "<< array->name() << endl;
172 for(
auto chunk:chunks){
173 auto current_offset = chunk.get_offset();
174 auto current_size = chunk.
get_size();
178 if(current_offset!=next_contiguous_chunk_offset){
180 unsigned long long gap_size = current_offset - next_contiguous_chunk_offset;
182 cout <<
"FOUND GAP current_offset: " << current_offset <<
183 " nbytes: " << current_offset <<
184 " next_contiguous_chunk_offset: " << next_contiguous_chunk_offset <<
185 " gap_size: " << gap_size <<
186 " currentSuperChunk.size(): " << currentSuperChunk->size() << endl;
190 if(!currentSuperChunk->empty()){
191 currentSuperChunk =
new vector<const Chunk *>();
192 super_chunks.push_back(currentSuperChunk);
196 currentSuperChunk->push_back(&chunk);
197 next_contiguous_chunk_offset = current_offset + current_size;
201 if(currentSuperChunk->empty()) {
202 super_chunks.pop_back();
203 delete currentSuperChunk;
205 cout <<
"SuperChunk Inventory For Array: " << array->name() << endl;
206 unsigned long long sc_count=0;
207 for(
auto super_chunk: super_chunks) {
208 cout <<
" SuperChunk[" << sc_count++ <<
"] contains : " << super_chunk->size() <<
" chunks."
211 for (
auto chunk:*super_chunk) {
212 cout <<
" " << chunk->to_string() << endl;
219 cerr << prolog <<
" ERROR! The variable: "<< var->name()
220 <<
" is not an instance of DmrppArray. SKIPPING"<< endl;
227 void inventory_super_chunks(libdap::D4Group *group,
bool only_constrained, vector<SuperChunk *> &super_chunks){
230 auto gtr = group->grp_begin();
231 while(gtr!=group->grp_end()){
232 if(debug) cout <<
"Found Group: "<< (*gtr)->name() << endl;
233 inventory_super_chunks(*gtr++, only_constrained, super_chunks);
237 auto vtr = group->var_begin();
238 while(vtr!=group->var_end()){
239 if(debug) cout <<
"Found Variable: "<< (*vtr)->type_name() <<
" " << (*vtr)->name() << endl;
240 compute_super_chunks(*vtr++, only_constrained, super_chunks);
245 void inventory_super_chunks(DMRpp &dmr,
bool only_constrained, vector<SuperChunk *> &super_chunks){
246 inventory_super_chunks(dmr.root(), only_constrained, super_chunks);
250 ifstream dmrpp_ifs (dmrpp_filename);
251 if (dmrpp_ifs.is_open())
255 auto dmr =
new DMRpp(&factory,dmrpp_filename);
256 parser.
intern(dmrpp_ifs, dmr);
260 throw BESInternalFatalError(
"The provided file could not be opened. filename: '"+dmrpp_filename+
"'",__FILE__,__LINE__);
264 void inventory_super_chunks(
const string dmrpp_filename){
265 cout <<
"DMR++ file: " << dmrpp_filename << endl;
268 vector<SuperChunk *> super_chunks;
273 dmrpp::inventory_super_chunks(*dmr,
false, super_chunks);
276 cout <<
"DMR++ file: " << dmrpp_filename << endl;
277 cout <<
"Produced " << super_chunks.size() <<
" SuperChunks." << endl;
278 for(
auto super_chunk: super_chunks) {
279 cout << super_chunk->to_string(
true) << endl;
285 void dump_vars(libdap::D4Group *group){
287 auto gtr = group->grp_begin();
288 while(gtr!=group->grp_end()){
289 if(debug) cout <<
"Found Group: "<< (*gtr)->name() << endl;
294 auto vtr = group->var_begin();
295 while(vtr!=group->var_end()){
296 libdap::BaseType *bt = *vtr++;
302 void dump_vars(DMRpp &dmr){
303 dump_vars(dmr.root());
307 int main(
int argc,
char *argv[]) {
308 string bes_log_file(
"superchunky_bes.log");
314 string cache_effective_urls(
"false");
315 char *prefixCstr = getenv(
"prefix");
322 cout <<
"bes_log_file: " << bes_log_file << endl;
335 BESIndent::SetIndent(
"");
337 for(
auto i=1; i<argc; i++){
338 string dmrpp_filename(argv[i]);
341 dmrpp::DMRpp *dmrpp = dmrpp::get_dmrpp( dmrpp_filename);
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual bool start(std::string name)
static std::string assemblePath(const std::string &firstPart, const std::string &secondPart, bool leadingSlash=false, bool trailingSlash=false)
Assemble path fragments making sure that they are separated by a single '/' character.
static TheBESKeys * TheKeys()
void set_key(const std::string &key, const std::string &val, bool addto=false)
allows the user to set key/value pairs from within the application.
static std::string ConfigFile
Provide a way to print the DMR++ response.
Extend libdap::Array so that a handler can read data using a DMR++ file.
virtual unsigned long long get_size(bool constrained=false)
Return the total number of elements in this Array.
virtual unsigned long add_chunk(std::shared_ptr< http::url > d_data_url, const std::string &byte_order, unsigned long long size, unsigned long long offset, const std::string &position_in_array)
Add a new chunk as defined by an h4:byteStream element.
virtual const std::vector< unsigned long long > & get_chunk_dimension_sizes() const
The chunk dimension sizes held in a const vector.
virtual const std::vector< std::shared_ptr< Chunk > > & get_immutable_chunks() const
A const reference to the vector of chunks.
void intern(std::istream &f, libdap::DMR *dest_dmr)