bes  Updated for version 3.20.10
FONcTransform.cc
1 // FONcTransform.cc
2 
3 // This file is part of BES Netcdf File Out Module
4 
5 // Copyright (c) 2004,2005 University Corporation for Atmospheric Research
6 // Author: Patrick West <pwest@ucar.edu> and Jose Garcia <jgarcia@ucar.edu>
7 //
8 // This library is free software; you can redistribute it and/or
9 // modify it under the terms of the GNU Lesser General Public
10 // License as published by the Free Software Foundation; either
11 // version 2.1 of the License, or (at your option) any later version.
12 //
13 // This library is distributed in the hope that it will be useful,
14 // but WITHOUT ANY WARRANTY; without even the implied warranty of
15 // MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16 // Lesser General Public License for more details.
17 //
18 // You should have received a copy of the GNU Lesser General Public
19 // License along with this library; if not, write to the Free Software
20 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21 //
22 // You can contact University Corporation for Atmospheric Research at
23 // 3080 Center Green Drive, Boulder, CO 80301
24 
25 // (c) COPYRIGHT University Corporation for Atmospheric Research 2004-2005
26 // Please read the full copyright statement in the file COPYRIGHT_UCAR.
27 //
28 // Authors:
29 // pwest Patrick West <pwest@ucar.edu>
30 // jgarcia Jose Garcia <jgarcia@ucar.edu>
31 // kyang Kent Yang <myang6@hdfgroup.org> (for DAP4/netCDF-4 enhancement)
32 // slloyd Samuel Lloyd <slloyd@opendap.org> (netCDF file streaming)
33 
34 #include "config.h"
35 
36 #include <sstream>
37 
38 #include <BESResponseObject.h>
39 #include <BESDapResponseBuilder.h>
40 #include <BESDataHandlerInterface.h>
41 #include <BESUtil.h>
42 #include <TempFile.h>
43 #include <BESDapNames.h>
44 #include <BESDataNames.h>
45 #include <BESDataDDSResponse.h>
46 #include <BESDMRResponse.h>
47 #include <BESRequestHandlerList.h>
48 #include <BESDapFunctionResponseCache.h>
49 
50 #include "FONcRequestHandler.h" // for the keys
51 
52 #include "FONcTransform.h"
53 #include "FONcUtils.h"
54 #include "FONcBaseType.h"
55 #include "FONcAttributes.h"
56 #include "FONcTransmitter.h"
57 #include "history_utils.h"
58 
59 #include <libdap/DDS.h>
60 #include <libdap/DMR.h>
61 #include <libdap/D4Group.h>
62 #include <libdap/D4Attributes.h>
63 #include <libdap/Structure.h>
64 #include <libdap/Array.h>
65 #include <libdap/Grid.h>
66 #include <libdap/Sequence.h>
67 #include <BESDebug.h>
68 #include <BESInternalError.h>
69 #include <BESInternalFatalError.h>
70 
71 #include "DapFunctionUtils.h"
72 
73 using std::ostringstream;
74 using std::istringstream;
75 
87 FONcTransform::FONcTransform(DDS *dds, BESDataHandlerInterface &dhi, const string &localfile, const string &ncVersion) :
88  _ncid(0), _dds(nullptr), _dmr(nullptr), d_obj(nullptr), d_dhi(nullptr)
89 {
90  if (!dds) {
91  string s = (string) "File out netcdf, " + "null DDS passed to constructor";
92  throw BESInternalError(s, __FILE__, __LINE__);
93  }
94  if (localfile.empty()) {
95  string s = (string) "File out netcdf, " + "empty local file name passed to constructor";
96  throw BESInternalError(s, __FILE__, __LINE__);
97  }
98  _localfile = localfile;
99  _dds = dds;
100  _returnAs = ncVersion;
101 
102  // if there is a variable, attribute, dimension name that is not
103  // compliant with netcdf naming conventions then we will create
104  // a new name. If the new name does not begin with an alpha
105  // character then we will prefix it with name_prefix. We will
106  // get this prefix from the type of data that we are reading in,
107  // such as nc, h4, h5, ff, jg, etc...
108  dhi.first_container();
109  if (dhi.container) {
111  }
112  else {
113  FONcUtils::name_prefix = "nc_";
114  }
115 }
116 
128 FONcTransform::FONcTransform(DMR *dmr, BESDataHandlerInterface &dhi, const string &localfile, const string &ncVersion) :
129  _ncid(0), _dds(nullptr), _dmr(nullptr), d_obj(nullptr), d_dhi(nullptr)
130 {
131  if (!dmr) {
132  string s = (string) "File out netcdf, null DMR passed to constructor";
133  throw BESInternalError(s, __FILE__, __LINE__);
134  }
135  if (localfile.empty()) {
136  string s = (string) "File out netcdf, " + "empty local file name passed to constructor";
137  throw BESInternalError(s, __FILE__, __LINE__);
138  }
139  _localfile = localfile;
140  _dmr = dmr;
141  _returnAs = ncVersion;
142 
143  // if there is a variable, attribute, dimension name that is not
144  // compliant with netcdf naming conventions then we will create
145  // a new name. If the new name does not begin with an alpha
146  // character then we will prefix it with name_prefix. We will
147  // get this prefix from the type of data that we are reading in,
148  // such as nc, h4, h5, ff, jg, etc...
149  dhi.first_container();
150  if (dhi.container) {
152  }
153  else {
154  FONcUtils::name_prefix = "nc_";
155  }
156 }
157 
170  const string &ncVersion) :
171  _ncid(0), _dds(nullptr), _dmr(nullptr), d_obj(obj), d_dhi(dhi), _localfile(localfile),
172  _returnAs(ncVersion)
173 {
174  if (!d_obj) {
175  string s = (string) "File out netcdf, " + "null BESResponseObject passed to constructor";
176  throw BESInternalError(s, __FILE__, __LINE__);
177  }
178  if (_localfile.empty()) {
179  string s = (string) "File out netcdf, " + "empty local file name passed to constructor";
180  throw BESInternalError(s, __FILE__, __LINE__);
181  }
182 
183  // if there is a variable, attribute, dimension name that is not
184  // compliant with netcdf naming conventions then we will create
185  // a new name. If the new name does not begin with an alpha
186  // character then we will prefix it with name_prefix. We will
187  // get this prefix from the type of data that we are reading in,
188  // such as nc, h4, h5, ff, jg, etc...
189  dhi->first_container();
190  if (dhi->container) {
192  }
193  else {
194  FONcUtils::name_prefix = "nc_";
195  }
196 }
197 
198 
204 {
205  for (auto &b: _fonc_vars) {
206  delete b;
207  }
208  for (auto &b: _total_fonc_vars_in_grp) {
209  delete b;
210  }
211  // _dmr is not managed by the BESDMRResponse class in this code. However
212  // _dds still is. jhrg 8/13/21
213  delete _dmr;
214 #if 0
215  bool done = false;
216  while (!done) {
217  vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
218  vector<FONcBaseType *>::iterator e = _fonc_vars.end();
219  if (i == e) {
220  done = true;
221  }
222  else {
223  // These are the FONc types, not the actual ones
224  FONcBaseType *b = (*i);
225  delete b;
226  _fonc_vars.erase(i);
227  }
228  }
229  done = false;
230  while (!done) {
231  vector<FONcBaseType *>::iterator i = _total_fonc_vars_in_grp.begin();
232  vector<FONcBaseType *>::iterator e = _total_fonc_vars_in_grp.end();
233  if (i == e) {
234  done = true;
235  }
236  else {
237  // These are the FONc types, not the actual ones
238  FONcBaseType *b = (*i);
239  delete b;
240  _total_fonc_vars_in_grp.erase(i);
241  }
242  }
243 #endif
244 }
245 
246 // previous transform() fct, keep for rollback purposes. sbl 5.14.21
247 #if 0
256 void FONcTransform::transform()
257 {
259 
260  // Convert the DDS into an internal format to keep track of
261  // variables, arrays, shared dimensions, grids, common maps,
262  // embedded structures. It only grabs the variables that are to be
263  // sent.
264  DDS::Vars_iter vi = _dds->var_begin();
265  DDS::Vars_iter ve = _dds->var_end();
266  for (; vi != ve; vi++) {
267  if ((*vi)->send_p()) {
268  BaseType *v = *vi;
269 
270  BESDEBUG("fonc", "FONcTransform::transform() - Converting variable '" << v->name() << "'" << endl);
271 
272  // This is a factory class call, and 'fg' is specialized for 'v'
273  FONcBaseType *fb = FONcUtils::convert(v,FONcTransform::_returnAs,FONcRequestHandler::classic_model);
274 #if 0
275  fb->setVersion( FONcTransform::_returnAs );
276  if ( FONcTransform::_returnAs == RETURNAS_NETCDF4 ) {
277  if (FONcRequestHandler::classic_model)
278  fb->setNC4DataModel("NC4_CLASSIC_MODEL");
279  else
280  fb->setNC4DataModel("NC4_ENHANCED");
281  }
282 #endif
283  _fonc_vars.push_back(fb);
284  vector<string> embed;
285  fb->convert(embed);
286  }
287  }
288 
289  // Open the file for writing
290  int stax;
291  if ( FONcTransform::_returnAs == RETURNAS_NETCDF4 ) {
292  if (FONcRequestHandler::classic_model){
293  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-4 cache file in classic mode. fileName: " << _localfile << endl);
294  stax = nc_create(_localfile.c_str(), NC_CLOBBER|NC_NETCDF4|NC_CLASSIC_MODEL, &_ncid);
295  }
296  else {
297  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
298  stax = nc_create(_localfile.c_str(), NC_CLOBBER|NC_NETCDF4, &_ncid);
299  }
300  }
301  else {
302  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-3 cache file. fileName: " << _localfile << endl);
303  stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
304  }
305 
306  if (stax != NC_NOERR) {
307  FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
308  }
309 
310  try {
311  // Here we will be defining the variables of the netcdf and
312  // adding attributes. To do this we must be in define mode.
313  nc_redef(_ncid);
314 
315  // For each converted FONc object, call define on it to define
316  // that object to the netcdf file. This also adds the attributes
317  // for the variables to the netcdf file
318  vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
319  vector<FONcBaseType *>::iterator e = _fonc_vars.end();
320  for (; i != e; i++) {
321  FONcBaseType *fbt = *i;
322  BESDEBUG("fonc", "FONcTransform::transform() - Defining variable: " << fbt->name() << endl);
323  fbt->define(_ncid);
324  }
325 
326  if(FONcRequestHandler::no_global_attrs == false) {
327  // Add any global attributes to the netcdf file
328  AttrTable &globals = _dds->get_attr_table();
329  BESDEBUG("fonc", "FONcTransform::transform() - Adding Global Attributes" << endl << globals << endl);
330  bool is_netCDF_enhanced = false;
331  if(FONcTransform::_returnAs == RETURNAS_NETCDF4 && FONcRequestHandler::classic_model==false)
332  is_netCDF_enhanced = true;
333  FONcAttributes::add_attributes(_ncid, NC_GLOBAL, globals, "", "",is_netCDF_enhanced);
334  }
335 
336  // We are done defining the variables, dimensions, and
337  // attributes of the netcdf file. End the define mode.
338  int stax = nc_enddef(_ncid);
339 
340  // Check error for nc_enddef. Handling of HDF failures
341  // can be detected here rather than later. KY 2012-10-25
342  if (stax != NC_NOERR) {
343  FONcUtils::handle_error(stax, "File out netcdf, unable to end the define mode: " + _localfile, __FILE__, __LINE__);
344  }
345 
346  // Write everything out
347  i = _fonc_vars.begin();
348  e = _fonc_vars.end();
349  for (; i != e; i++) {
350  FONcBaseType *fbt = *i;
351  BESDEBUG("fonc", "FONcTransform::transform() - Writing data for variable: " << fbt->name() << endl);
352  fbt->write(_ncid);
353  }
354 
355  stax = nc_close(_ncid);
356  if (stax != NC_NOERR)
357  FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
358  }
359  catch (BESError &e) {
360  (void) nc_close(_ncid); // ignore the error at this point
361  throw;
362  }
363 }
364 #endif
365 
374 void FONcTransform::transform_dap2(ostream &strm)
375 {
376 #if 0
377  BESDapResponseBuilder responseBuilder;
378  // Use the DDS from the ResponseObject along with the parameters
379  // from the DataHandlerInterface to load the DDS with values.
380  // Note that the BESResponseObject will manage the loaded_dds object's
381  // memory. Make this a shared_ptr<>. jhrg 9/6/16
382 #endif
383  // Now that we are ready to start reading the response data we
384  // cancel any pending timeout alarm according to the configuration.
386 
387  BESDEBUG("fonc", "FONcTransmitter::send_data() - Reading data into DataDDS" << endl);
388 
390 
391  d_dhi->first_container();
392 
393  auto bdds = dynamic_cast<BESDataDDSResponse *>(d_obj);
394  if (!bdds) throw BESInternalFatalError("Expected a BESDataDDSResponse instance", __FILE__, __LINE__);
395 
396  _dds = bdds->get_dds();
397 
398  BESDapResponseBuilder besDRB;
399 
400  besDRB.set_dataset_name(_dds->filename());
401  besDRB.set_ce(d_dhi->data[POST_CONSTRAINT]);
402  besDRB.set_async_accepted(d_dhi->data[ASYNC]);
403  besDRB.set_store_result(d_dhi->data[STORE_RESULT]);
404 
405 
406  // This function is used by all fileout modules and they need to include the attributes in data access.
407  // So obtain the attributes if necessary. KY 2019-10-30
408  if (bdds->get_ia_flag() == false) {
409  BESRequestHandler *besRH = BESRequestHandlerList::TheList()->find_handler(
410  d_dhi->container->get_container_type());
411  besRH->add_attributes(*d_dhi);
412  }
413 
414  ConstraintEvaluator &eval = bdds->get_ce();
415 
416  // Split constraint into two halves; stores the function and non-function parts in this instance.
417  besDRB.split_ce(eval);
418  // If there are functions, parse them and eval.
419  // Use that DDS and parse the non-function ce
420  // Serialize using the second ce and the second dds
421  if (!besDRB.get_btp_func_ce().empty()) {
422  BESDEBUG("fonc","Found function(s) in CE: " << besDRB.get_btp_func_ce() << endl);
423 
424  BESDapFunctionResponseCache *responseCache = BESDapFunctionResponseCache::get_instance();
425 
426  ConstraintEvaluator func_eval;
427  DDS *fdds = nullptr;
428  if (responseCache && responseCache->can_be_cached(_dds, besDRB.get_btp_func_ce())) {
429  fdds = responseCache->get_or_cache_dataset(_dds, besDRB.get_btp_func_ce());
430  }
431  else {
432  func_eval.parse_constraint(besDRB.get_btp_func_ce(), *_dds);
433  fdds = func_eval.eval_function_clauses(*_dds);
434  }
435 
436  delete _dds; // Delete so that we can ...
437  bdds->set_dds(fdds); // Transfer management responsibility
438  _dds = fdds;
439 
440  // Server functions might mark (i.e. setting send_p) so variables will use their read()
441  // methods. Clear that so the CE in d_dap2ce will control what is
442  // sent. If that is empty (there was only a function call) all
443  // of the variables in the intermediate DDS (i.e., the function
444  // result) will be sent.
445  _dds->mark_all(false);
446 
447  // Look for one or more top level Structures whose name indicates (by way of ending with
448  // "_uwrap") that their contents should be moved to the top level.
449  //
450  // This is in support of a hack around the current API where server side functions
451  // may only return a single DAP object and not a collection of objects. The name suffix
452  // "_unwrap" is used as a signal from the function to the the various response
453  // builders and transmitters that the representation needs to be altered before
454  // transmission, and that in fact is what happens in our friend
455  // promote_function_output_structures()
456  promote_function_output_structures(_dds);
457  }
458 
459  // evaluate the rest of the CE - the part that follows the function calls.
460  eval.parse_constraint(besDRB.get_ce(), *_dds);
461 
462  _dds->tag_nested_sequences(); // Tag Sequences as Parent or Leaf node.
463 
464  //throw_if_dap2_response_too_big(_dds); // TODO Fix this, sbl 5/6/21
465 
466  // Convert the DDS into an internal format to keep track of
467  // variables, arrays, shared dimensions, grids, common maps,
468  // embedded structures. It only grabs the variables that are to be
469  // sent.
470  for (auto vi = _dds->var_begin(), ve = _dds->var_end(); vi != ve; vi++) {
471  if ((*vi)->send_p()) {
472  BESDEBUG("fonc", "FONcTransform::transform() - Converting variable '" << (*vi)->name() << "'" << endl);
473 
474  // This is a factory class call, and 'fg' is specialized for '*vi'
475  FONcBaseType *fb = FONcUtils::convert((*vi), FONcTransform::_returnAs, FONcRequestHandler::classic_model);
476 
477  _fonc_vars.push_back(fb);
478  vector<string> embed;
479  fb->convert(embed);
480  }
481  }
482 
483  updateHistoryAttributes(_dds, d_dhi->data[POST_CONSTRAINT]);
484 
485  // Open the file for writing
486  int stax;
487  if (FONcTransform::_returnAs == RETURN_AS_NETCDF4) {
488  if (FONcRequestHandler::classic_model) {
489  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-4 cache file in classic mode. fileName: "
490  << _localfile << endl);
491  stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL, &_ncid);
492  }
493  else {
494  BESDEBUG("fonc",
495  "FONcTransform::transform() - Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
496  stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
497  }
498  }
499  else {
500  BESDEBUG("fonc", "FONcTransform::transform() - Opening NetCDF-3 cache file. fileName: " << _localfile << endl);
501  stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
502  }
503 
504  if (stax != NC_NOERR) {
505  FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
506  }
507 
508  int current_fill_prop_vaule;
509 
510  stax = nc_set_fill(_ncid, NC_NOFILL, &current_fill_prop_vaule);
511  if (stax != NC_NOERR) {
512  FONcUtils::handle_error(stax, "File out netcdf, unable to set fill to NC_NOFILL: " + _localfile, __FILE__, __LINE__);
513  }
514 
515  try {
516  // Here we will be defining the variables of the netcdf and
517  // adding attributes. To do this we must be in define mode.
518  nc_redef(_ncid);
519 
520  // For each converted FONc object, call define on it to define
521  // that object to the netcdf file. This also adds the attributes
522  // for the variables to the netcdf file
523  for (FONcBaseType *fbt: _fonc_vars) {
524  BESDEBUG("fonc", "FONcTransform::transform() - Defining variable: " << fbt->name() << endl);
525  fbt->define(_ncid);
526  }
527 
528  if (FONcRequestHandler::no_global_attrs == false) {
529  // Add any global attributes to the netcdf file
530  AttrTable &globals = _dds->get_attr_table();
531  BESDEBUG("fonc", "FONcTransform::transform() - Adding Global Attributes" << endl << globals << endl);
532  bool is_netCDF_enhanced = false;
533  if (FONcTransform::_returnAs == RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model == false)
534  is_netCDF_enhanced = true;
535  FONcAttributes::add_attributes(_ncid, NC_GLOBAL, globals, "", "", is_netCDF_enhanced);
536  }
537 
538  // We are done defining the variables, dimensions, and
539  // attributes of the netcdf file. End the define mode.
540  int stax = nc_enddef(_ncid);
541 
542  // Check error for nc_enddef. Handling of HDF failures
543  // can be detected here rather than later. KY 2012-10-25
544  if (stax != NC_NOERR) {
545  FONcUtils::handle_error(stax, "File out netcdf, unable to end the define mode: " + _localfile, __FILE__,
546  __LINE__);
547  }
548  // write file data
549  uint64_t byteCount = 0;
550 
551  if (is_streamable()) {
552  byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
553  BESDEBUG("fonc", "FONcTransform::transform() - first write data to stream, count: " << byteCount << endl);
554  }
555 
556  for (FONcBaseType *fbt: _fonc_vars) {
557  BESDEBUG("fonc", "FONcTransform::transform() - Writing data for variable: " << fbt->name() << endl);
558 
559  fbt->set_dds(_dds);
560  fbt->set_eval(&eval);
561 
562  fbt->write(_ncid);
563  nc_sync(_ncid);
564 
565 
566  if (is_streamable()) {
567  // write the whats been written
568  byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
569  BESDEBUG("fonc", "FONcTransform::transform() - Writing data to stream, count: " << byteCount << endl);
570  }
571  }
572 
573  stax = nc_close(_ncid);
574  if (stax != NC_NOERR)
575  FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
576 
577  byteCount = BESUtil::file_to_stream_helper(_localfile, strm, byteCount);
578  BESDEBUG("fonc", "FONcTransform::transform() - after nc_close() count: " << byteCount << endl);
579  }
580  catch (BESError &e) {
581  (void) nc_close(_ncid); // ignore the error at this point
582  throw;
583  }
584 }
585 
593 bool FONcTransform::is_streamable(){
594  if (FONcTransform::_returnAs == RETURN_AS_NETCDF4){
595  return false;
596  }
597 
598  if (_dds != nullptr){
599  return is_dds_streamable();
600  }
601  else {
602  return is_dmr_streamable(_dmr->root());
603  }
604 }
605 
611 bool FONcTransform::is_dds_streamable(){
612  for (auto var = _dds->var_begin(), varEnd = _dds->var_end(); var != varEnd; ++var) {
613  if ((*var)->type() == dods_structure_c) {
614  return false; // cannot be streamed
615  }
616  }
617  return true;
618 }
619 
626 bool FONcTransform::is_dmr_streamable(D4Group *group){
627  for (auto var = group->var_begin(), varEnd = group->var_end(); var != varEnd; ++var) {
628  if ((*var)->type() == dods_structure_c)
629  return false ; // cannot be streamed
630 
631  if ((*var)->type() == dods_group_c) {
632  D4Group *g = dynamic_cast<D4Group *>(*var);
633  if (g != nullptr && !is_dmr_streamable(g)) {
634  return false;
635  }
636  }
637  }
638  return true;
639 }
640 
641 
651 {
653  BESDEBUG("fonc", "FONcTransform::transform_dap4() Reading data into DataDMR" << endl);
654 
656 
657  d_dhi->first_container();
658 
659  BESDapResponseBuilder responseBuilder;
660  _dmr = responseBuilder.setup_dap4_intern_data(d_obj, *d_dhi).release();
661 
662  BESDapResponseBuilder besDRB;
663 
664  besDRB.set_dataset_name(_dmr->filename());
665 
666  // Added set of DAP4 fields. jhrg 5/30/21
667  besDRB.set_dap4ce(d_dhi->data[DAP4_CONSTRAINT]);
668  besDRB.set_dap4function(d_dhi->data[DAP4_FUNCTION]);
669 
670  besDRB.set_async_accepted(d_dhi->data[ASYNC]);
671  besDRB.set_store_result(d_dhi->data[STORE_RESULT]);
672 
673  // Convert the DMR into an internal format to keep track of
674  // variables, arrays, shared dimensions, grids, common maps,
675  // embedded structures. It only grabs the variables that are to be
676  // sent.
677 
678  BESDEBUG("fonc", "Coming into transform_dap4() " << endl);
679 
680  // First check if this DMR has groups etc.
681  bool support_group = check_group_support();
682 
683  if (true == support_group) {
684 
685  int stax = -1;
686  BESDEBUG("fonc",
687  "FONcTransform::transform_dap4() - Opening NetCDF-4 cache file. fileName: " << _localfile << endl);
688  stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
689  if (stax != NC_NOERR)
690  FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
691 
692  D4Group *root_grp = _dmr->root();
693 
694  // Declare the dimname to dimid map to handle netCDF-4 dimensions
695  map<string, int> fdimname_to_id;
696 
697  // Generate a list of the groups in the final netCDF file.
698  // The attributes of these groups should be included.
699  gen_included_grp_list(root_grp);
700 
701 #if !NDEBUG
702  for (std::set<string>::iterator it=_included_grp_names.begin(); it!=_included_grp_names.end(); ++it)
703  BESDEBUG("fonc","included group list name is: "<<*it<<endl);
704 #endif
705  // Build a global dimension name table for all variables if
706  // the constraint is not empty!
707  check_and_obtain_dimensions(root_grp, true);
708 
709  // Don't remove the following code, they are for debugging.
710 #if !NDEBUG
711  map<string,unsigned long>:: iterator it;
712 
713  for(it=GFQN_dimname_to_dimsize.begin();it!=GFQN_dimname_to_dimsize.end();++it) {
714  BESDEBUG("fonc", "Final GFQN dim name is: "<<it->first<<endl);
715  BESDEBUG("fonc", "Final GFQN dim size is: "<<it->second<<endl);
716  }
717 
718  for(it=VFQN_dimname_to_dimsize.begin();it!=VFQN_dimname_to_dimsize.end();++it) {
719  BESDEBUG("fonc", "Final VFQN dim name is: "<<it->first<<endl);
720  BESDEBUG("fonc", "Final VFQN dim size is: "<<it->second<<endl);
721  }
722 #endif
723 
724  // DAP4 requires the DAP4 dimension sizes defined in the group should be changed
725  // according to the corresponding variable sizes. Check section 8.6.2 at
726  // https://docs.opendap.org/index.php/DAP4:_Specification_Volume_1
727  //
728  map<string, unsigned long>::iterator git, vit;
729  for (git = GFQN_dimname_to_dimsize.begin(); git != GFQN_dimname_to_dimsize.end(); ++git) {
730  for (vit = VFQN_dimname_to_dimsize.begin(); vit != VFQN_dimname_to_dimsize.end(); ++vit) {
731  if (git->first == vit->first) {
732  if (git->second != vit->second)
733  git->second = vit->second;
734  break;
735  }
736  }
737  }
738 
739  // This part of code is to address the possible dimension name conflict
740  // when variables in the constraint don't have dimension names. Fileout netCDF
741  // adds the fake dimensions such as dim1, dim2...to these variables.
742  // If these dimension names are used by
743  // the file to be handled, the dimension conflict will corrupt the final output.
744  // The idea is to find if there are any dimension names like dim1, dim2 ...
745  // under the root group.
746  // We will remember them and not use these names as fake dimension names.
747  //
748  // Obtain the dim. names under the root group
749  vector<string> root_d4_dimname_list;
750  for (git = GFQN_dimname_to_dimsize.begin(); git != GFQN_dimname_to_dimsize.end(); ++git) {
751  string d4_temp_dimname = git->first.substr(1);
752  //BESDEBUG("fonc", "d4_temp_dimname: "<<d4_temp_dimname<<endl);
753  if (d4_temp_dimname.find('/') == string::npos)
754  root_d4_dimname_list.push_back(d4_temp_dimname);
755  }
756 
757 #if !NDEBUG
758  for(unsigned int i = 0; i <root_d4_dimname_list.size();i++)
759  BESDEBUG("fonc", "root_d4 dim name is: "<<root_d4_dimname_list[i]<<endl);
760 #endif
761 
762  // Only remember the root dimension names that are like "dim1,dim2,..."
763  vector<int> root_dim_suffix_nums;
764  for (unsigned int i = 0; i < root_d4_dimname_list.size(); i++) {
765  if (root_d4_dimname_list[i].size() < 4)
766  continue;
767  else if (root_d4_dimname_list[i].substr(0, 3) != "dim")
768  continue;
769  else {
770  string temp_suffix = root_d4_dimname_list[i].substr(3);
771  //BESDEBUG("fonc", "temp_suffix: "<<temp_suffix<<endl);
772  bool ignored_suffix = false;
773  for (unsigned int j = 0; j < temp_suffix.size(); j++) {
774  if (!isdigit(temp_suffix[j])) {
775  ignored_suffix = true;
776  break;
777  }
778  }
779  if (ignored_suffix == true)
780  continue;
781  else
782  root_dim_suffix_nums.push_back(atoi(temp_suffix.c_str()));
783  }
784  }
785 
786 #if !NDEBUG
787  for(unsigned int i = 0; i <root_dim_suffix_nums.size();i++)
788  BESDEBUG("fonc", "root_dim_suffix_nums: "<<root_dim_suffix_nums[i]<<endl);
789 
790 
791  for(it=GFQN_dimname_to_dimsize.begin();it!=GFQN_dimname_to_dimsize.end();++it) {
792  BESDEBUG("fonc", "RFinal GFQN dim name is: "<<it->first<<endl);
793  BESDEBUG("fonc", "RFinal GFQN dim size is: "<<it->second<<endl);
794  }
795 
796  for(it=VFQN_dimname_to_dimsize.begin();it!=VFQN_dimname_to_dimsize.end();++it) {
797  BESDEBUG("fonc", "RFinal VFQN dim name is: "<<it->first<<endl);
798  BESDEBUG("fonc", "RFinal VFQN dim size is: "<<it->second<<endl);
799  }
800 #endif
801 
802  // Now we transform all the objects(including groups) to netCDF-4
803  transform_dap4_group(root_grp, true, _ncid, fdimname_to_id, root_dim_suffix_nums);
804  stax = nc_close(_ncid);
805  if (stax != NC_NOERR)
806  FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
807 
808  }
809  else // No group, handle as the classic way
810  transform_dap4_no_group();
811 
812  return;
813 }
814 
820 void FONcTransform::transform_dap4_no_group()
821 {
822 
823  D4Group *root_grp = _dmr->root();
824 #if !NDEBUG
825  D4Dimensions *root_dims = root_grp->dims();
826  for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
827  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
828  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
829  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
830  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
831  }
832 #endif
833  Constructor::Vars_iter vi = root_grp->var_begin();
834  Constructor::Vars_iter ve = root_grp->var_end();
835 
836  for (; vi != ve; vi++) {
837  if ((*vi)->send_p()) {
838  BaseType *v = *vi;
839 
840  BESDEBUG("fonc",
841  "FONcTransform::transform_dap4_no_group() - Converting variable '" << v->name() << "'" << endl);
842 
843  // This is a factory class call, and 'fg' is specialized for 'v'
844  FONcBaseType *fb = FONcUtils::convert(v, FONcTransform::_returnAs, FONcRequestHandler::classic_model);
845  _fonc_vars.push_back(fb);
846 
847  vector<string> embed;
848  fb->convert(embed,true,false);
849  }
850  }
851 
852 #if !NDEBUG
853  if(root_grp->grp_begin() == root_grp->grp_end())
854  BESDEBUG("fonc", "FONcTransform::transform_dap4() - No group " << endl);
855  else
856  BESDEBUG("fonc", "FONcTransform::transform_dap4() - has group " << endl);
857  for (D4Group::groupsIter gi = root_grp->grp_begin(), ge = root_grp->grp_end(); gi != ge; ++gi)
858  BESDEBUG("fonc", "FONcTransform::transform_dap4() - group name: " << (*gi)->name() << endl);
859 #endif
860 
861  updateHistoryAttributes(_dmr, d_dhi->data[POST_CONSTRAINT]);
862 
863  // Open the file for writing
864  int stax = -1;
865  if (FONcTransform::_returnAs == RETURN_AS_NETCDF4) {
866  if (FONcRequestHandler::classic_model) {
867  BESDEBUG("fonc",
868  "FONcTransform::transform_dap4_no_group() - Opening NetCDF-4 cache file in classic mode. fileName: "
869  << _localfile << endl);
870  stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4 | NC_CLASSIC_MODEL, &_ncid);
871  }
872  else {
873  BESDEBUG("fonc",
874  "FONcTransform::transform_dap4_no_group() - Opening NetCDF-4 cache file. fileName: " << _localfile
875  << endl);
876  stax = nc_create(_localfile.c_str(), NC_CLOBBER | NC_NETCDF4, &_ncid);
877  }
878  }
879  else {
880  BESDEBUG("fonc",
881  "FONcTransform::transform_dap4_no_group() - Opening NetCDF-3 cache file. fileName: " << _localfile
882  << endl);
883  stax = nc_create(_localfile.c_str(), NC_CLOBBER, &_ncid);
884  }
885 
886  if (stax != NC_NOERR) {
887  FONcUtils::handle_error(stax, "File out netcdf, unable to open: " + _localfile, __FILE__, __LINE__);
888  }
889 
890  try {
891  // Here we will be defining the variables of the netcdf and
892  // adding attributes. To do this we must be in define mode.
893  nc_redef(_ncid);
894 
895  // For each converted FONc object, call define on it to define
896  // that object to the netcdf file. This also adds the attributes
897  // for the variables to the netcdf file
898  vector<FONcBaseType *>::iterator i = _fonc_vars.begin();
899  vector<FONcBaseType *>::iterator e = _fonc_vars.end();
900  for (; i != e; i++) {
901  FONcBaseType *fbt = *i;
902  BESDEBUG("fonc", "FONcTransform::transform_dap4_no_group() - Defining variable: " << fbt->name() << endl);
903  //fbt->set_is_dap4(true);
904  fbt->define(_ncid);
905  }
906 
907  if (FONcRequestHandler::no_global_attrs == false) {
908 
909  // Add any global attributes to the netcdf file
910  D4Group *root_grp = _dmr->root();
911  D4Attributes *d4_attrs = root_grp->attributes();
912 
913  BESDEBUG("fonc",
914  "FONcTransform::transform_dap4_no_group() handle GLOBAL DAP4 attributes " << d4_attrs << endl);
915 #if !NDEBUG
916  for (D4Attributes::D4AttributesIter ii = d4_attrs->attribute_begin(), ee = d4_attrs->attribute_end(); ii != ee; ++ii) {
917  string name = (*ii)->name();
918  BESDEBUG("fonc", "FONcTransform::transform_dap4() GLOBAL attribute name is "<<name <<endl);
919  }
920 #endif
921  bool is_netCDF_enhanced = false;
922  if (FONcTransform::_returnAs == RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model == false)
923  is_netCDF_enhanced = true;
924  FONcAttributes::add_dap4_attributes(_ncid, NC_GLOBAL, d4_attrs, "", "", is_netCDF_enhanced);
925  }
926 
927  // We are done defining the variables, dimensions, and
928  // attributes of the netcdf file. End the define mode.
929  int stax = nc_enddef(_ncid);
930 
931  // Check error for nc_enddef. Handling of HDF failures
932  // can be detected here rather than later. KY 2012-10-25
933  if (stax != NC_NOERR) {
934  FONcUtils::handle_error(stax, "File out netcdf, unable to end the define mode: " + _localfile, __FILE__,
935  __LINE__);
936  }
937 
938  // Write everything out
939  i = _fonc_vars.begin();
940  e = _fonc_vars.end();
941  for (; i != e; i++) {
942  FONcBaseType *fbt = *i;
943  BESDEBUG("fonc",
944  "FONcTransform::transform_dap4_no_group() - Writing data for variable: " << fbt->name() << endl);
945  fbt->write(_ncid);
946  }
947 
948  stax = nc_close(_ncid);
949  if (stax != NC_NOERR)
950  FONcUtils::handle_error(stax, "File out netcdf, unable to close: " + _localfile, __FILE__, __LINE__);
951  }
952  catch (BESError &e) {
953  (void) nc_close(_ncid); // ignore the error at this point
954  throw;
955  }
956 
957 }
958 
959 // Transform the DMR to a netCDF-4 file when there are DAP4 groups.
960 void FONcTransform::transform_dap4_group(D4Group *grp,
961  bool is_root_grp,
962  int par_grp_id, map<string, int> &fdimname_to_id,
963  vector<int> &root_dim_suffix_nums)
964 {
965 
966  bool included_grp = false;
967 
968  if (_dmr->get_ce_empty()) {
969  BESDEBUG("fonc",
970  "Check-get_ce_empty. FONcTransform::transform_dap4() in group - group name: " << grp->FQN() << endl);
971  included_grp = true;
972  }
973  // Always include the root and its attributes.
974  else if (is_root_grp == true)
975  included_grp = true;
976  else {
977  // Check if this group is in the group list kept in the file.
978  set<string>::iterator iset;
979  if (_included_grp_names.find(grp->FQN()) != _included_grp_names.end())
980  included_grp = true;
981  }
982 
983  // Call the internal routine to transform the DMR that has groups if this group is in the group list..
984  // If this group is not in the group list, we know all its subgroups are also not in the list, just stop and return.
985  if (included_grp == true)
986  transform_dap4_group_internal(grp, is_root_grp, par_grp_id, fdimname_to_id, root_dim_suffix_nums);
987  return;
988 }
989 
998 void FONcTransform::transform_dap4_group_internal(D4Group *grp,
999  bool is_root_grp,
1000  int par_grp_id, map<string, int> &fdimname_to_id,
1001  vector<int> &rds_nums)
1002 {
1003 
1004  BESDEBUG("fonc", "transform_dap4_group_internal() - inside" << endl);
1005  int grp_id = -1;
1006  int stax = -1;
1007 
1008  updateHistoryAttributes(_dmr, d_dhi->data[POST_CONSTRAINT]);
1009 
1010  if (is_root_grp == true)
1011  grp_id = _ncid;
1012  else {
1013  stax = nc_def_grp(par_grp_id, (*grp).name().c_str(), &grp_id);
1014  BESDEBUG("fonc", "transform_dap4_group_internal() - group name is " << (*grp).name() << endl);
1015  if (stax != NC_NOERR)
1016  FONcUtils::handle_error(stax, "File out netcdf, unable to define group: " + _localfile, __FILE__, __LINE__);
1017 
1018  }
1019 
1020  D4Dimensions *grp_dims = grp->dims();
1021  for (D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
1022 
1023 #if !NDEBUG
1024  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1025  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1026  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1027  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1028 #endif
1029 
1030  unsigned long dimsize = (*di)->size();
1031 
1032  // The dimension size may need to be updated because of the expression constraint.
1033  map<string, unsigned long>::iterator it;
1034  for (it = GFQN_dimname_to_dimsize.begin(); it != GFQN_dimname_to_dimsize.end(); ++it) {
1035  if (it->first == (*di)->fully_qualified_name())
1036  dimsize = it->second;
1037  }
1038 
1039  // Define dimension.
1040  int g_dimid = -1;
1041  stax = nc_def_dim(grp_id, (*di)->name().c_str(), dimsize, &g_dimid);
1042  if (stax != NC_NOERR)
1043  FONcUtils::handle_error(stax, "File out netcdf, unable to define dimension: " + _localfile, __FILE__,
1044  __LINE__);
1045  // Save this dimension ID in a map.
1046  fdimname_to_id[(*di)->fully_qualified_name()] = g_dimid;
1047  }
1048 
1049  Constructor::Vars_iter vi = grp->var_begin();
1050  Constructor::Vars_iter ve = grp->var_end();
1051 
1052  vector<FONcBaseType *> fonc_vars_in_grp;
1053  for (; vi != ve; vi++) {
1054  if ((*vi)->send_p()) {
1055  BaseType *v = *vi;
1056 
1057  BESDEBUG("fonc",
1058  "FONcTransform::transform_dap4_group() - Converting variable '" << v->name() << "'" << endl);
1059 
1060  // This is a factory class call, and 'fg' is specialized for 'v'
1061  //FONcBaseType *fb = FONcUtils::convert(v,FONcTransform::_returnAs,FONcRequestHandler::classic_model);
1062  FONcBaseType *fb = FONcUtils::convert(v, RETURN_AS_NETCDF4, false, fdimname_to_id, rds_nums);
1063 
1064  fonc_vars_in_grp.push_back(fb);
1065 
1066  // This is needed to avoid the memory leak.
1067  _total_fonc_vars_in_grp.push_back(fb);
1068 
1069  vector<string> embed;
1070  fb->convert(embed, true,true);
1071  }
1072  }
1073 
1074 #if !NDEBUG
1075  if(grp->grp_begin() == grp->grp_end())
1076  BESDEBUG("fonc", "FONcTransform::transform_dap4() - No group " << endl);
1077  else
1078  BESDEBUG("fonc", "FONcTransform::transform_dap4() - has group " << endl);
1079 #endif
1080 
1081 
1082  try {
1083  // Here we will be defining the variables of the netcdf and
1084  // adding attributes. To do this we must be in define mode.
1085  //nc_redef(_ncid);
1086 
1087  vector<FONcBaseType *>::iterator i = fonc_vars_in_grp.begin();
1088  vector<FONcBaseType *>::iterator e = fonc_vars_in_grp.end();
1089  for (; i != e; i++) {
1090  FONcBaseType *fbt = *i;
1091  BESDEBUG("fonc", "FONcTransform::transform_dap4_group() - Defining variable: " << fbt->name() << endl);
1092  //fbt->set_is_dap4(true);
1093  fbt->define(grp_id);
1094  }
1095 
1096  bool is_netCDF_enhanced = false;
1097  if (FONcTransform::_returnAs == RETURN_AS_NETCDF4 && FONcRequestHandler::classic_model == false)
1098  is_netCDF_enhanced = true;
1099 
1100 
1101  bool add_attr = true;
1102 
1103  // Only the root attribute may be ignored.
1104  if (FONcRequestHandler::no_global_attrs == true && is_root_grp == true)
1105  add_attr = false;
1106 
1107  if (true == add_attr) {
1108  D4Attributes *d4_attrs = grp->attributes();
1109  BESDEBUG("fonc", "FONcTransform::transform_dap4_group() - Adding Group Attributes" << endl);
1110  // add dap4 group attributes.
1111  FONcAttributes::add_dap4_attributes(grp_id, NC_GLOBAL, d4_attrs, "", "", is_netCDF_enhanced);
1112  }
1113 
1114  // Write every variable in this group.
1115  i = fonc_vars_in_grp.begin();
1116  e = fonc_vars_in_grp.end();
1117  for (; i != e; i++) {
1118  FONcBaseType *fbt = *i;
1119  BESDEBUG("fonc",
1120  "FONcTransform::transform_dap4_group() - Writing data for variable: " << fbt->name() << endl);
1121  //fbt->write(_ncid);
1122  fbt->write(grp_id);
1123  }
1124 
1125  // Now handle all the child groups.
1126  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1127  BESDEBUG("fonc", "FONcTransform::transform_dap4() in group - group name: " << (*gi)->name() << endl);
1128  transform_dap4_group(*gi, false, grp_id, fdimname_to_id, rds_nums);
1129  }
1130 
1131  }
1132  catch (BESError &e) {
1133  (void) nc_close(_ncid); // ignore the error at this point
1134  throw;
1135  }
1136 
1137 }
1138 
1139 
1140 // Group support is only on when netCDF-4 is in enhanced model and there are groups in the DMR.
1141 bool FONcTransform::check_group_support()
1142 {
1143  if (RETURN_AS_NETCDF4 == FONcTransform::_returnAs && false == FONcRequestHandler::classic_model &&
1144  (_dmr->root()->grp_begin() != _dmr->root()->grp_end()))
1145  return true;
1146  else
1147  return false;
1148 }
1149 
1150 // Generate the final group list in the netCDF-4 file. Empty groups and their attributes will be removed.
1151 void FONcTransform::gen_included_grp_list(D4Group *grp)
1152 {
1153  bool grp_has_var = false;
1154  if (grp) {
1155  BESDEBUG("fnoc", "<coming to the D4 group has name " << grp->name() << endl);
1156  BESDEBUG("fnoc", "<coming to the D4 group has fullpath " << grp->FQN() << endl);
1157 
1158  if (grp->var_begin() != grp->var_end()) {
1159 
1160  BESDEBUG("fnoc", "<has the vars " << endl);
1161  Constructor::Vars_iter vi = grp->var_begin();
1162  Constructor::Vars_iter ve = grp->var_end();
1163 
1164  for (; vi != ve; vi++) {
1165 
1166  // This variable is selected(in the local constraints).
1167  if ((*vi)->send_p()) {
1168  grp_has_var = true;
1169 
1170  //If a var in this group is selected, we need to include this group in the netcdf-4 file.
1171  //We always include root attributes, so no need to obtain grp_names for the root.
1172  if (grp->FQN() != "/")
1173  _included_grp_names.insert(grp->FQN());
1174  break;
1175  }
1176  }
1177  }
1178  // Loop through the subgroups to build up the list.
1179  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1180  BESDEBUG("fonc", "obtain included groups - group name: " << (*gi)->name() << endl);
1181  gen_included_grp_list(*gi);
1182  }
1183  }
1184 
1185  // If this group is in the final list, all its ancestors(except root, since it is always selected),should also be included.
1186  if (grp_has_var == true) {
1187  D4Group *temp_grp = grp;
1188  while (temp_grp) {
1189  if (temp_grp->get_parent()) {
1190  temp_grp = static_cast<D4Group *>(temp_grp->get_parent());
1191  if (temp_grp->FQN() != "/")
1192  _included_grp_names.insert(temp_grp->FQN());
1193  }
1194  else
1195  temp_grp = 0;
1196  }
1197  }
1198 
1199 }
1200 
1201 void FONcTransform::check_and_obtain_dimensions(D4Group *grp, bool is_root_grp)
1202 {
1203 
1204  // We may not need to do this way,it may overkill.
1205  bool included_grp = false;
1206 
1207  if (_dmr->get_ce_empty())
1208  included_grp = true;
1209  // Always include the root attributes.
1210  else if (is_root_grp == true)
1211  included_grp = true;
1212  else {
1213  // Check if this group is in the group list kept in the file.
1214  set<string>::iterator iset;
1215  if (_included_grp_names.find(grp->FQN()) != _included_grp_names.end())
1216  included_grp = true;
1217  }
1218 
1219  if (included_grp == true)
1220  check_and_obtain_dimensions_internal(grp);
1221 }
1222 
1223 void FONcTransform::check_and_obtain_dimensions_internal(D4Group *grp)
1224 {
1225 
1226  // Remember the Group Fully Qualified dimension Name and the corresponding dimension size.
1227  D4Dimensions *grp_dims = grp->dims();
1228  if (grp_dims) {
1229  for (D4Dimensions::D4DimensionsIter di = grp_dims->dim_begin(), de = grp_dims->dim_end(); di != de; ++di) {
1230 
1231 #if !NDEBUG
1232  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1233  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1234  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1235  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1236 #endif
1237  unsigned long dimsize = (*di)->size();
1238  if ((*di)->constrained()) {
1239  dimsize = ((*di)->c_stop() - (*di)->c_start()) / (*di)->c_stride() + 1;
1240 
1241  }
1242  GFQN_dimname_to_dimsize[(*di)->fully_qualified_name()] = dimsize;
1243  }
1244  }
1245 
1246  // The size of DAP4 dimension needs to be updated if the dimension size of a variable with the same dimension is
1247  // different. So we also need to remember the Variable FQN dimension name and size.
1248  // Check section 8.6.2 of DAP4 specification(https://docs.opendap.org/index.php/DAP4:_Specification_Volume_1)
1249  Constructor::Vars_iter vi = grp->var_begin();
1250  Constructor::Vars_iter ve = grp->var_end();
1251  for (; vi != ve; vi++) {
1252  if ((*vi)->send_p()) {
1253  if ((*vi)->is_vector_type()) {
1254  Array *t_a = dynamic_cast<Array *>(*vi);
1255  Array::Dim_iter dim_i = t_a->dim_begin();
1256  Array::Dim_iter dim_e = t_a->dim_end();
1257  for (; dim_i != dim_e; dim_i++) {
1258  if ((*dim_i).name != "") {
1259  D4Dimension *d4dim = t_a->dimension_D4dim(dim_i);
1260  if (d4dim) {
1261  BESDEBUG("fonc", "transform_dap4() check dim- dim name is: " << d4dim->name() << endl);
1262  BESDEBUG("fonc", "transform_dap4() check dim- dim size is: " << d4dim->size() << endl);
1263  BESDEBUG("fonc", "transform_dap4() check dim- fully_qualfied_dim name is: "
1264  << d4dim->fully_qualified_name() << endl);
1265 
1266  unsigned long dimsize = t_a->dimension_size(dim_i, true);
1267 #if !NDEBUG
1268  BESDEBUG("fonc", "transform_dap4() check dim- final dim size is: "<< dimsize << endl);
1269 #endif
1270  pair<map<string, unsigned long>::iterator, bool> ret_it;
1271  ret_it = VFQN_dimname_to_dimsize.insert(
1272  pair<string, unsigned long>(d4dim->fully_qualified_name(), dimsize));
1273  if (ret_it.second == false && ret_it.first->second != dimsize) {
1274  string err = "fileout_netcdf-4: dimension found with the same name, but different size";
1275  throw BESInternalError(err, __FILE__, __LINE__);
1276  }
1277  //VFQN_dimname_to_dimsize[d4dim->fully_qualified_name()] = dimsize;
1278  }
1279  else
1280  throw BESInternalError("Has dimension name but D4 dimension is NULL", __FILE__, __LINE__);
1281  }
1282  // No need to handle the case when the dimension name doesn't exist. This will be handled in FONcArray.cc.
1283  // else { }
1284  }
1285  }
1286  }
1287  }
1288 
1289 #if !NDEBUG
1290  map<string,unsigned long>:: iterator it;
1291  for(it=GFQN_dimname_to_dimsize.begin();it!=GFQN_dimname_to_dimsize.end();++it) {
1292  BESDEBUG("fonc", "GFQN dim name is: "<<it->first<<endl);
1293  BESDEBUG("fonc", "GFQN dim size is: "<<it->second<<endl);
1294  }
1295 
1296  for(it=VFQN_dimname_to_dimsize.begin();it!=VFQN_dimname_to_dimsize.end();++it) {
1297  BESDEBUG("fonc", "VFQN dim name is: "<<it->first<<endl);
1298  BESDEBUG("fonc", "VFQN dim size is: "<<it->second<<endl);
1299  }
1300 
1301 #endif
1302 
1303  // Go through all the descendent groups.
1304  for (D4Group::groupsIter gi = grp->grp_begin(), ge = grp->grp_end(); gi != ge; ++gi) {
1305  BESDEBUG("fonc",
1306  "FONcTransform::check_and_obtain_dimensions() in group - group name: " << (*gi)->name() << endl);
1307  check_and_obtain_dimensions(*gi, false);
1308  }
1309 
1310 }
1311 
1321 void FONcTransform::dump(ostream &strm) const
1322 {
1323  strm << BESIndent::LMarg << "FONcTransform::dump - (" << (void *) this << ")" << endl;
1324  BESIndent::Indent();
1325  strm << BESIndent::LMarg << "ncid = " << _ncid << endl;
1326  strm << BESIndent::LMarg << "temporary file = " << _localfile << endl;
1327  BESIndent::Indent();
1328  vector<FONcBaseType *>::const_iterator i = _fonc_vars.begin();
1329  vector<FONcBaseType *>::const_iterator e = _fonc_vars.end();
1330  for (; i != e; i++) {
1331  FONcBaseType *fbt = *i;
1332  fbt->dump(strm);
1333  }
1334  BESIndent::UnIndent();
1335  BESIndent::UnIndent();
1336 }
1337 
1338 
std::string get_container_type() const
retrieve the type of data this container holds, such as cedar or netcdf.
Definition: BESContainer.h:232
Cache the results from server functions.
virtual libdap::DDS * get_or_cache_dataset(libdap::DDS *dds, const std::string &constraint)
Return a DDS loaded with data that can be serialized back to a client.
virtual void set_dataset_name(const std::string _dataset)
Set the dataset pathname.
virtual void split_ce(libdap::ConstraintEvaluator &eval, const std::string &expr="")
virtual std::string get_ce() const
Get the constraint expression.
virtual void set_dap4ce(std::string _ce)
virtual void set_dap4function(std::string _func)
virtual void set_ce(std::string _ce)
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
Structure storing information used by the BES to handle the request.
std::map< std::string, std::string > data
the map of string data that will be required for the current request.
void first_container()
set the container pointer to the first container in the containers list
BESContainer * container
pointer to current container in this interface
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
virtual BESRequestHandler * find_handler(const std::string &handler_name)
find and return the specified request handler
Represents a specific data type request handler.
Abstract base class representing a specific set of information in response to a request to the BES.
static void conditional_timeout_cancel()
Checks if the timeout alarm should be canceled based on the value of the BES key BES....
Definition: BESUtil.cc:992
static void add_dap4_attributes(int ncid, int varid, D4Attributes *d4_attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
add_dap4_attributes
static void add_attributes(int ncid, int varid, AttrTable &attrs, const string &var_name, const string &prepend_attr, bool is_netCDF_enhanced)
helper function for add_attributes
A DAP BaseType with file out netcdf information included.
Definition: FONcBaseType.h:64
virtual void define(int ncid)
Define the variable in the netcdf file.
Definition: FONcBaseType.cc:60
virtual void setNC4DataModel(const string &nc4_datamodel)
Identifies the netCDF4 data model (CLASSIC or ENHANCED)
virtual void setVersion(const std::string &version)
Identifies variable with use of NetCDF4 features.
virtual void dump(std::ostream &strm) const =0
dump the contents of this object to the specified ostream
FONcTransform(DDS *dds, BESDataHandlerInterface &dhi, const string &localfile, const string &netcdfVersion="netcdf")
virtual void dump(ostream &strm) const
dumps information about this transformation object for debugging purposes
virtual void transform_dap2(ostream &strm)
Transforms each of the variables of the DataDDS to the NetCDF file.
virtual void transform_dap4()
Transforms each of the variables of the DMR to the NetCDF file.
virtual ~FONcTransform()
Destructor.
static void handle_error(int stax, const string &err, const string &file, int line)
handle any netcdf errors
Definition: FONcUtils.cc:424
static string name_prefix
If a variable name, dimension name, or attribute name begins with a character that is not supported b...
Definition: FONcUtils.h:60
static void reset()
Resets the FONc transformation for a new input and out file.
Definition: FONcUtils.cc:68