bes  Updated for version 3.20.10
HDF5RequestHandler.cc
Go to the documentation of this file.
1 
2 // -*- mode: c++; c-basic-offset:4 -*-
3 
4 // This file is part of hdf5_handler, a data handler for the OPeNDAP data
5 // server.
6 
7 // Copyright (c) 2002,2003 OPeNDAP, Inc.
8 // Copyright (c) 2007-2016 The HDF Group, Inc. and OPeNDAP, Inc.
9 // Author: James Gallagher <jgallagher@opendap.org>
10 //
11 // This is free software; you can redistribute it and/or modify it under the
12 // terms of the GNU Lesser General Public License as published by the Free
13 // Software Foundation; either version 2.1 of the License, or (at your
14 // option) any later version.
15 //
16 // This software is distributed in the hope that it will be useful, but
17 // WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY
18 // or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
19 // License for more details.
20 //
21 // You should have received a copy of the GNU Lesser General Public
22 // License along with this library; if not, write to the Free Software
23 // Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
24 //
25 // You can contact OPeNDAP, Inc. at PO Box 112, Saunderstown, RI. 02874-0112.
26 
27 // HDF5RequestHandler.cc
33 
34 
35 #include <iostream>
36 #include <fstream>
37 #include <sstream>
38 #include <string>
39 #include <sys/types.h>
40 #include <sys/stat.h>
41 #include <fcntl.h>
42 #include <unistd.h>
43 
44 #include <libdap/DMR.h>
45 #include <libdap/D4BaseTypeFactory.h>
46 #include <BESDMRResponse.h>
47 #include <ObjMemCache.h>
48 #include "HDF5_DMR.h"
49 
50 #include <libdap/mime_util.h>
51 #include "hdf5_handler.h"
52 #include "HDF5RequestHandler.h"
53 #include "HDF5_DDS.h"
54 
55 #include <BESDASResponse.h>
56 #include <BESDDSResponse.h>
57 #include <BESDataDDSResponse.h>
58 #include <libdap/Ancillary.h>
59 #include <BESInfo.h>
60 #include <BESDapNames.h>
61 #include <BESResponseNames.h>
62 #include <BESContainer.h>
63 #include <BESResponseHandler.h>
64 #include <BESVersionInfo.h>
65 #include <BESServiceRegistry.h>
66 #include <BESUtil.h>
67 #include <BESDapError.h>
68 #include <BESInternalFatalError.h>
69 #include <TheBESKeys.h>
70 #include <BESDebug.h>
71 #include <BESStopWatch.h>
72 #include "h5get.h"
73 #include "config_hdf5.h"
74 
75 #define HDF5_NAME "h5"
76 #include "h5cfdaputil.h"
77 
78 using namespace std;
79 using namespace libdap;
80 
81 #define prolog std::string("HDF5RequestHandler::").append(__func__).append("() - ")
82 
83 // The debug function to dump all the contents of a DAS table.
84 void get_attr_contents(AttrTable* temp_table);
85 
86 // Five functions to generate a DAS cache file.
87 // 1. The wrapper function to call write_das_table_to_file to generate the cache.
88 void write_das_to_file(DAS*das_ptr,FILE* das_file);
89 
90 // 2. The main function to generate the DAS cache.
91 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file);
92 
93 // 3. The function to write the DAS container/table name to the cache
94 void write_container_name_to_file(const string&,FILE* das_file);
95 
96 // 4. The function to write the DAS attribute name,type and values to the cache
97 void write_das_attr_info(AttrTable* dtp,const string&,const string&,FILE * das_file);
98 
99 // 5. The function to copy a string to a memory buffer.
100 char* copy_str(char *temp_ptr,const string & str);
101 
102 // These two functions are for reading the DAS from the DAS cache file
103 // 1. Obtain the string from a memory buffer.
104 char* obtain_str(char*temp_ptr,string & str);
105 
106 // 2. The main function to obtain DAS info from the cache.
107 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at);
108 
109 // Check if the BES key is set.
110 //bool check_and_set_beskeys(const string);
111 
112 // Obtain the BES key as an integer
113 static unsigned int get_uint_key(const string &key,unsigned int def_val);
114 static unsigned long get_ulong_key(const string &key,unsigned long def_val);
115 
116 // Obtain the BES key as a floating-pointer number.
117 static float get_float_key(const string &key, float def_val);
118 
119 // Obtain the BES key as a string.
120 static string get_beskeys(const string&);
121 
122 //static bool is_beskeys_set_true(const string &);
123 bool obtain_beskeys_info(const string &, bool &);
124 
125 // For the CF option
126 extern void read_cfdas(DAS &das, const string & filename,hid_t fileid);
127 extern void read_cfdds(DDS &dds, const string & filename,hid_t fileid);
128 extern void read_cfdmr(DMR *dmr, const string & filename,hid_t fileid);
129 
130 
131 
132 // Check the description of cache_entries and cache_purge_level at h5.conf.in.
133 unsigned int HDF5RequestHandler::_mdcache_entries = 500;
134 unsigned int HDF5RequestHandler::_lrdcache_entries = 0;
135 unsigned int HDF5RequestHandler::_srdcache_entries = 0;
136 float HDF5RequestHandler::_cache_purge_level = 0.2;
137 
138 // Metadata object cache at DAS,DDS and DMR.
139 ObjMemCache *HDF5RequestHandler::das_cache = 0;
140 ObjMemCache *HDF5RequestHandler::dds_cache = 0;
141 ObjMemCache *HDF5RequestHandler::datadds_cache = 0;
142 ObjMemCache *HDF5RequestHandler::dmr_cache = 0;
143 
144 ObjMemCache *HDF5RequestHandler::lrdata_mem_cache = 0;
145 ObjMemCache *HDF5RequestHandler::srdata_mem_cache = 0;
146 
147 // Set default values of all BES keys according to h5.conf.in.
148 // This will help the generation of DMRPP. No need to
149 // set multiple keys if the user's setting is the same as
150 // the h5.conf.in. KY 2021-08-23
151 bool HDF5RequestHandler::_usecf = true;
152 bool HDF5RequestHandler::_pass_fileid = false;
153 bool HDF5RequestHandler::_disable_structmeta = true;
154 bool HDF5RequestHandler::_disable_ecsmeta = false;
155 bool HDF5RequestHandler::_keep_var_leading_underscore = false;
156 bool HDF5RequestHandler::_check_name_clashing = false;
157 bool HDF5RequestHandler::_add_path_attrs = true;
158 bool HDF5RequestHandler::_drop_long_string = true;
159 bool HDF5RequestHandler::_fillvalue_check = true;
160 bool HDF5RequestHandler::_check_ignore_obj = false;
161 bool HDF5RequestHandler::_flatten_coor_attr = true;
162 bool HDF5RequestHandler::_default_handle_dimension = true; //Ignored when _usecf=true.
163 bool HDF5RequestHandler::_eos5_rm_convention_attr_path = true;
164 bool HDF5RequestHandler::_dmr_long_int = true;
165 bool HDF5RequestHandler::_no_zero_size_fullnameattr = false;
166 bool HDF5RequestHandler::_enable_coord_attr_add_path = true;
167 
168 bool HDF5RequestHandler::_usecfdmr = true;
169 
170 bool HDF5RequestHandler::_common_cache_dirs = false;
171 
172 bool HDF5RequestHandler::_use_disk_cache =false;
173 bool HDF5RequestHandler::_use_disk_dds_cache =false;
174 string HDF5RequestHandler::_disk_cache_dir ="";
175 string HDF5RequestHandler::_disk_cachefile_prefix ="";
176 unsigned long long HDF5RequestHandler::_disk_cache_size =0;
177 
178 
179 bool HDF5RequestHandler::_disk_cache_comp_data =false;
180 bool HDF5RequestHandler::_disk_cache_float_only_comp_data =false;
181 float HDF5RequestHandler::_disk_cache_comp_threshold =1.0;
182 unsigned long HDF5RequestHandler::_disk_cache_var_size =0;
183 
184 bool HDF5RequestHandler::_use_disk_meta_cache = false;
185 string HDF5RequestHandler::_disk_meta_cache_path ="";
186 
187 bool HDF5RequestHandler::_use_latlon_disk_cache = false;
188 long HDF5RequestHandler::_latlon_disk_cache_size =0;
189 string HDF5RequestHandler::_latlon_disk_cache_dir ="";
190 string HDF5RequestHandler::_latlon_disk_cachefile_prefix="";
191 
192 DMR* HDF5RequestHandler::dmr_int64 = 0;
193 
194 
195 #if 0
196 //BaseTypeFactory factory;
197 //libdap::DDS HDF5RequestHandler::hd_dds(&factory,"");
198 #endif
199 string HDF5RequestHandler::_stp_east_filename;
200 string HDF5RequestHandler::_stp_north_filename;
201 vector<string> HDF5RequestHandler::lrd_cache_dir_list;
202 vector<string> HDF5RequestHandler::lrd_non_cache_dir_list;
203 vector<string> HDF5RequestHandler::lrd_var_cache_file_list;
204 
205 #if 0
206 //libdap::DDS*cache_dds;
207 #endif
208 
209 HDF5RequestHandler::HDF5RequestHandler(const string & name)
210  :BESRequestHandler(name)
211 {
212 
213  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
214 
215  add_handler(DAS_RESPONSE, HDF5RequestHandler::hdf5_build_das);
216  add_handler(DDS_RESPONSE, HDF5RequestHandler::hdf5_build_dds);
217  add_handler(DATA_RESPONSE, HDF5RequestHandler::hdf5_build_data);
218  add_handler(DMR_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
219  add_handler(DAP4DATA_RESPONSE, HDF5RequestHandler::hdf5_build_dmr);
220 
221  add_handler(HELP_RESPONSE, HDF5RequestHandler::hdf5_build_help);
222  add_handler(VERS_RESPONSE, HDF5RequestHandler::hdf5_build_version);
223 
224 #if !(DYNAMIC_CONFIG_ENABLED)
225  load_config();
226 #endif
227 
228  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
229 }
230 
231 HDF5RequestHandler::~HDF5RequestHandler()
232 {
233  // delete the cache.
234  delete das_cache;
235  delete dds_cache;
236  delete datadds_cache;
237  delete dmr_cache;
238  delete lrdata_mem_cache;
239  delete srdata_mem_cache;
240 
241 }
242 
246 void HDF5RequestHandler::load_config()
247 {
248  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
249  BESStopWatch sw;
250  if (BESISDEBUG(HDF5_NAME)){
251  sw.start(prolog,"ClockTheBESKeys");
252  }
253 
254  // Obtain the metadata cache entries and purge level.
255  HDF5RequestHandler::_mdcache_entries = get_uint_key("H5.MetaDataMemCacheEntries", 0);
256  HDF5RequestHandler::_lrdcache_entries = get_uint_key("H5.LargeDataMemCacheEntries", 0);
257  HDF5RequestHandler::_srdcache_entries = get_uint_key("H5.SmallDataMemCacheEntries", 0);
258  HDF5RequestHandler::_cache_purge_level = get_float_key("H5.CachePurgeLevel", 0.2);
259 
260  if (get_mdcache_entries()) { // else it stays at its default of null
261  das_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
262  dds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
263  datadds_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
264  dmr_cache = new ObjMemCache(get_mdcache_entries(), get_cache_purge_level());
265  }
266 
267  // Starting from hyrax 1.16.5, users don't need to explicitly set the BES keys if
268  // they are happy with the default settings in the h5.conf.in.
269  // In the previous releases, we required users to set BES key values. Otherwise,
270  // the BES key values of true/false will always be false if the key is not found.
271  // Hopefully this change will make it convenient for the general users.
272  // In the mean time, the explicit BES key settings will not be affected.
273  // KY 2021-10-13
274  //
275  // Check if the EnableCF key is set.
276  bool has_key = false;
277  bool key_value = obtain_beskeys_info("H5.EnableCF",has_key);
278  if(has_key)
279  _usecf = key_value;
280  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCF: " << (_usecf?"true":"false") << endl);
281 
282  // The DefaultHandleDimension is effective only when EnableCF=false
283  //_default_handle_dimension = check_and_set_beskeys("H5.DefaultHandleDimension");
284  key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
285  if(has_key)
286  _default_handle_dimension = key_value;
287  BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
288 
289  // The following keys are only effective when EnableCF is true or unset(EnableCF is true if users don't set the key).
290  //_pass_fileid = obtain_beskeys_info("H5.EnablePassFileID",has_key);
291  key_value = obtain_beskeys_info("H5.EnablePassFileID",has_key);
292  if(has_key)
293  _pass_fileid = key_value;
294  BESDEBUG(HDF5_NAME, prolog << "H5.EnablePassFileID: " << (_pass_fileid?"true":"false") << endl);
295 
296  //_disable_structmeta = check_and_set_beskeys("H5.DisableStructMetaAttr");
297  key_value = obtain_beskeys_info("H5.DisableStructMetaAttr",has_key);
298  if(has_key)
299  _disable_structmeta = key_value;
300  BESDEBUG(HDF5_NAME, prolog << "H5.DisableStructMetaAttr: " << (_disable_structmeta?"true":"false") << endl);
301 
302  //_disable_ecsmeta = check_and_set_beskeys("H5.DisableECSMetaAttr");
303  key_value = obtain_beskeys_info("H5.DisableECSMetaAttr",has_key);
304  if(has_key)
305  _disable_ecsmeta = key_value;
306  BESDEBUG(HDF5_NAME, prolog << "H5.DisableECSMetaAttr: " << (_disable_ecsmeta?"true":"false") << endl);
307 
308  //_keep_var_leading_underscore = check_and_set_beskeys("H5.KeepVarLeadingUnderscore");
309  key_value = obtain_beskeys_info("H5.KeepVarLeadingUnderscore",has_key);
310  if(has_key)
311  _keep_var_leading_underscore = key_value;
312  BESDEBUG(HDF5_NAME, prolog << "H5.KeepVarLeadingUnderscore: " << (_keep_var_leading_underscore?"true":"false") << endl);
313 
314  //_check_name_clashing = check_and_set_beskeys("H5.EnableCheckNameClashing");
315  key_value = obtain_beskeys_info("H5.EnableCheckNameClashing",has_key);
316  if(has_key)
317  _check_name_clashing = key_value;
318  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCheckNameClashing: " << (_check_name_clashing?"true":"false") << endl);
319 
320  //_add_path_attrs = check_and_set_beskeys("H5.EnableAddPathAttrs");
321  key_value = obtain_beskeys_info("H5.EnableAddPathAttrs",has_key);
322  if(has_key)
323  _add_path_attrs = key_value;
324  BESDEBUG(HDF5_NAME, prolog << "H5.EnableAddPathAttrs: " << (_add_path_attrs?"true":"false") << endl);
325 
326  //_drop_long_string = check_and_set_beskeys("H5.EnableDropLongString");
327  key_value = obtain_beskeys_info("H5.EnableDropLongString",has_key);
328  if(has_key)
329  _drop_long_string = key_value;
330  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDropLongString: " << (_drop_long_string?"true":"false") << endl);
331 
332  //_fillvalue_check = check_and_set_beskeys("H5.EnableFillValueCheck");
333  key_value = obtain_beskeys_info("H5.EnableFillValueCheck",has_key);
334  if(has_key)
335  _fillvalue_check = key_value;
336  BESDEBUG(HDF5_NAME, prolog << "H5.EnableFillValueCheck: " << (_fillvalue_check?"true":"false") << endl);
337 
338 
339  //_check_ignore_obj = check_and_set_beskeys("H5.CheckIgnoreObj");
340  key_value = obtain_beskeys_info("H5.CheckIgnoreObj",has_key);
341  if(has_key)
342  _check_ignore_obj = key_value;
343  BESDEBUG(HDF5_NAME, prolog << "H5.CheckIgnoreObj: " << (_check_ignore_obj?"true":"false") << endl);
344 
345  //_flatten_coor_attr = check_and_set_beskeys("H5.ForceFlattenNDCoorAttr");
346  key_value = obtain_beskeys_info("H5.ForceFlattenNDCoorAttr",has_key);
347  if(has_key)
348  _flatten_coor_attr = key_value;
349  BESDEBUG(HDF5_NAME, prolog << "H5.ForceFlattenNDCoorAttr: " << (_flatten_coor_attr?"true":"false") << endl);
350 
351 
352  //_eos5_rm_convention_attr_path= check_and_set_beskeys("H5.RmConventionAttrPath");
353  key_value = obtain_beskeys_info("H5.RmConventionAttrPath",has_key);
354  if(has_key)
355  _eos5_rm_convention_attr_path = key_value;
356  BESDEBUG(HDF5_NAME, prolog << "H5.RmConventionAttrPath: " << (_eos5_rm_convention_attr_path?"true":"false") << endl);
357 
358  //_dmr_long_int = check_and_set_beskeys("H5.EnableDMR64bitInt");
359  key_value = obtain_beskeys_info("H5.EnableDMR64bitInt",has_key);
360  if(has_key)
361  _dmr_long_int = key_value;
362  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDMR64bitInt: " << (_dmr_long_int?"true":"false") << endl);
363 
364  //_no_zero_size_fullnameattr = check_and_set_beskeys("H5.NoZeroSizeFullnameAttr");
365  key_value = obtain_beskeys_info("H5.NoZeroSizeFullnameAttr",has_key);
366  if(has_key)
367  _no_zero_size_fullnameattr = key_value;
368  BESDEBUG(HDF5_NAME, prolog << "H5.NoZeroSizeFullnameAttr: " << (_no_zero_size_fullnameattr?"true":"false") << endl);
369 
370  //_enable_coord_attr_add_path = check_and_set_beskeys("H5.EnableCoorattrAddPath");
371  key_value = obtain_beskeys_info("H5.EnableCoorattrAddPath",has_key);
372  if(has_key)
373  _enable_coord_attr_add_path = key_value;
374  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCoorattrAddPath: " << (_enable_coord_attr_add_path?"true":"false") << endl);
375 
376  //_usecfdmr = check_and_set_beskeys("H5.EnableCFDMR");
377  key_value = obtain_beskeys_info("H5.EnableCFDMR",has_key);
378  if(has_key)
379  _usecfdmr = key_value;
380  BESDEBUG(HDF5_NAME, prolog << "H5.EnableCFDMR: " << (_usecfdmr?"true":"false") << endl);
381 
382  //_use_disk_cache = check_and_set_beskeys("H5.EnableDiskDataCache");
383  key_value = obtain_beskeys_info("H5.EnableDiskDataCache",has_key);
384  if(has_key)
385  _use_disk_cache = key_value;
386  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDataCache: " << (_use_disk_cache?"true":"false") << endl);
387  _disk_cache_dir = get_beskeys("H5.DiskCacheDataPath");
388  _disk_cachefile_prefix = get_beskeys("H5.DiskCacheFilePrefix");
389  _disk_cache_size = get_ulong_key("H5.DiskCacheSize",0);
390 
391  //_disk_cache_comp_data = check_and_set_beskeys("H5.DiskCacheComp");
392  key_value = obtain_beskeys_info("H5.DiskCacheComp",has_key);
393  if(has_key)
394  _disk_cache_comp_data = key_value;
395  BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheComp: " << (_disk_cache_comp_data?"true":"false") << endl);
396 
397  //_disk_cache_float_only_comp_data = check_and_set_beskeys("H5.DiskCacheFloatOnlyComp");
398  key_value = obtain_beskeys_info("H5.DiskCacheFloatOnlyComp",has_key);
399  if(has_key)
400  _disk_cache_float_only_comp_data = key_value;
401  BESDEBUG(HDF5_NAME, prolog << "H5.DiskCacheFloatOnlyComp: " << (_disk_cache_float_only_comp_data?"true":"false") << endl);
402  _disk_cache_comp_threshold = get_float_key("H5.DiskCacheCompThreshold",1.0);
403  _disk_cache_var_size = 1024*get_uint_key("H5.DiskCacheCompVarSize",0);
404 
405  //_use_disk_meta_cache = check_and_set_beskeys("H5.EnableDiskMetaDataCache");
406  key_value = obtain_beskeys_info("H5.EnableDiskMetaDataCache",has_key);
407  if(has_key)
408  _use_disk_meta_cache = key_value;
409  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskMetaDataCache: " << (_use_disk_meta_cache?"true":"false") << endl);
410 
411  //_use_disk_dds_cache = check_and_set_beskeys("H5.EnableDiskDDSCache");
412  key_value = obtain_beskeys_info("H5.EnableDiskDDSCache",has_key);
413  if(has_key)
414  _use_disk_dds_cache = key_value;
415  BESDEBUG(HDF5_NAME, prolog << "H5.EnableDiskDDSCache: " << (_use_disk_dds_cache?"true":"false") << endl);
416  _disk_meta_cache_path = get_beskeys("H5.DiskMetaDataCachePath");
417 
418  //_use_latlon_disk_cache = check_and_set_beskeys("H5.EnableEOSGeoCacheFile");
419  key_value = obtain_beskeys_info("H5.EnableEOSGeoCacheFile",has_key);
420  if(has_key)
421  _use_latlon_disk_cache = key_value;
422  BESDEBUG(HDF5_NAME, prolog << "H5.EnableEOSGeoCacheFile: " << (_use_latlon_disk_cache?"true":"false") << endl);
423  _latlon_disk_cache_size = get_uint_key("H5.Cache.latlon.size",0);
424  _latlon_disk_cache_dir = get_beskeys("H5.Cache.latlon.path");
425  _latlon_disk_cachefile_prefix= get_beskeys("H5.Cache.latlon.prefix");
426 
427 #if 0
428  //_default_handle_dimension = check_and_set_beskeys("H5.DefaultHandleDimension");
429  key_value = obtain_beskeys_info("H5.DefaultHandleDimension",has_key);
430  if(has_key)
431  _default_handle_dimension = key_value;
432  BESDEBUG(HDF5_NAME, prolog << "H5.DefaultHandleDimension: " << (_default_handle_dimension?"true":"false") << endl);
433 #endif
434 
435  if(get_usecf()) {
436  if(get_lrdcache_entries()) {
437  lrdata_mem_cache = new ObjMemCache(get_lrdcache_entries(), get_cache_purge_level());
438  bool has_LFMC_config = false;
439  key_value = obtain_beskeys_info("H5.LargeDataMemCacheConfig",has_key);
440  if(has_key)
441  has_LFMC_config = key_value;
442  BESDEBUG(HDF5_NAME, prolog << "H5.LargeDataMemCacheConfig: " << (has_LFMC_config?"true":"false") << endl);
443  if(true == has_LFMC_config) {
444  _common_cache_dirs =obtain_lrd_common_cache_dirs();
445 #if 0
446  if(false == _common_cache_dirs)
447 cerr<<"No specific cache info"<<endl;
448 #endif
449  }
450  }
451  if(get_srdcache_entries()) {
452 
453  BESDEBUG(HDF5_NAME, prolog << "Generate memory cache for smaller coordinate variables" << endl);
454  srdata_mem_cache = new ObjMemCache(get_srdcache_entries(),get_cache_purge_level());
455 
456  }
457 
458  if(_disk_cache_comp_data == true && _use_disk_cache == true) {
459  if(_disk_cache_comp_threshold < 1.0) {
460  ostringstream ss;
461  ss<< _disk_cache_comp_threshold;
462  string _comp_threshold_str(ss.str());
463  string invalid_comp_threshold ="The Compression Threshold is the total size of the variable array";
464  invalid_comp_threshold+=" divided by the storage size of compressed array. It should always be >1";
465  invalid_comp_threshold+=" The current threhold set at h5.conf is ";
466  invalid_comp_threshold+=_comp_threshold_str;
467  invalid_comp_threshold+=" . Go back to h5.conf and change the H5.DiskCacheCompThreshold to a >1.0 number.";
468  throw BESInternalError(invalid_comp_threshold,__FILE__,__LINE__);
469  }
470  }
471  _stp_east_filename = get_beskeys("H5.STPEastFileName");
472  _stp_north_filename = get_beskeys("H5.STPNorthFileName");
473  }
474  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
475 }
476 
477 
478 // Build DAS
479 bool HDF5RequestHandler::hdf5_build_das(BESDataHandlerInterface & dhi)
480 {
481  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
482 #if DYNAMIC_CONFIG_ENABLED
483  load_config();
484 #endif
485 
486  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
487  hid_t cf_fileid = -1;
488 
489  // Obtain the HDF5 file name.
490  string filename = dhi.container->access();
491 
492  // Obtain the BES object from the client
493  BESResponseObject *response = dhi.response_handler->get_response_object() ;
494 
495  // Convert to the BES DAS response
496  BESDASResponse *bdas = dynamic_cast < BESDASResponse * >(response) ;
497  if( !bdas )
498  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
499 
500  try {
501  bdas->set_container( dhi.container->get_symbolic_name() ) ;
502  DAS *das = bdas->get_das();
503 
504  // Look inside the memory cache to see if it's initialized
505  DAS *cached_das_ptr = 0;
506  bool use_das_cache = false;
507  if (das_cache)
508  cached_das_ptr = static_cast<DAS*>(das_cache->get(filename));
509  if (cached_das_ptr)
510  use_das_cache = true;
511 
512  if (true == use_das_cache) {
513 
514  // copy the cached DAS into the BES response object
515  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
516  *das = *cached_das_ptr;
517  }
518  else {
519 
520  bool das_from_dc = false;
521  string das_cache_fname;
522 
523  // If the use_disk_meta_cache is set, check if the cache file exists and sets the flag.
524  if(_use_disk_meta_cache == true) {
525 
526  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
527  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
528 
529  if(access(das_cache_fname.c_str(),F_OK) !=-1)
530  das_from_dc = true;
531 
532  }
533 
534  // If reading DAS from the disk cache, call the corresponding function
535  if(true == das_from_dc) {
536  read_das_from_disk_cache(das_cache_fname,das);
537 
538  // If the memory cache is set, adding the DAS copy to the memory cache
539  if (das_cache) {
540  // add a copy
541  BESDEBUG(HDF5_NAME, prolog << "HDF5 DAS reading DAS from the disk cache. For memory cache, DAS added to the cache for : " << filename << endl);
542  das_cache->add(new DAS(*das), filename);
543  }
544  }
545 
546  else {// Need to build from the HDF5 file
547  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
548  if (true == _usecf) {//CF option
549 
550  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
551  if (cf_fileid < 0){
552  string invalid_file_msg="Could not open this HDF5 file ";
553  invalid_file_msg +=filename;
554  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
555  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
556  invalid_file_msg +=" distributor.";
557  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
558  }
559  // Need to check if DAP4 DMR CF 64-bit integer mapping is on.
560  if(HDF5RequestHandler::get_dmr_64bit_int()!=NULL)
561  HDF5RequestHandler::set_dmr_64bit_int(NULL);
562  read_cfdas( *das,filename,cf_fileid);
563  H5Fclose(cf_fileid);
564  }
565  else {// Default option
566  hid_t fileid = get_fileid(filename.c_str());
567  if (fileid < 0) {
568  string invalid_file_msg="Could not open this HDF5 file ";
569  invalid_file_msg +=filename;
570  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
571  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
572  invalid_file_msg +=" distributor.";
573  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
574 
575  }
576 
577  find_gloattr(fileid, *das);
578  depth_first(fileid, "/", *das);
579  close_fileid(fileid);
580  }
581 
582  Ancillary::read_ancillary_das( *das, filename ) ;
583 
584 #if 0
585 // Dump all attribute contents
586 AttrTable* top_table = das->get_top_level_attributes();
587 get_attr_contents(top_table);
588 
589 // Dump all variable contents
590 AttrTable::Attr_iter start_aiter=das->var_begin();
591 AttrTable::Attr_iter it = start_aiter;
592 AttrTable::Attr_iter end_aiter = das->var_end();
593 while(it != end_aiter) {
594 AttrTable* temp_table = das->get_table(it);
595 if(temp_table!=0){
596 cerr<<"var_begin"<<endl;
597 temp_table->print(cerr);
598 }
599 ++it;
600 }
601 #endif
602  // If the memory cache is turned on
603  if(das_cache) {
604  // add a copy
605  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
606  das_cache->add(new DAS(*das), filename);
607  }
608 
609  // DAS disk cache fname will be set only when the metadata disk cache is turned on
610  // So if it comes here, the das cache should be generated.
611  if(das_cache_fname!="") {
612  BESDEBUG(HDF5_NAME, prolog << "HDF5 Build DAS: Write DAS to disk cache " << das_cache_fname << endl);
613  write_das_to_disk_cache(das_cache_fname,das);
614  }
615  }
616  }
617 
618  bdas->clear_container() ;
619  }
620  catch(BESError & e) {
621  if(cf_fileid !=-1)
622  H5Fclose(cf_fileid);
623  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
624  throw;
625  }
626  catch(InternalErr & e) {
627 
628  if(cf_fileid !=-1)
629  H5Fclose(cf_fileid);
630  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
631  __FILE__, __LINE__);
632  }
633  catch(Error & e) {
634 
635  if(cf_fileid !=-1)
636  H5Fclose(cf_fileid);
637  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
638  __FILE__, __LINE__);
639  }
640  catch(...) {
641 
642  if(cf_fileid !=-1)
643  H5Fclose(cf_fileid);
644  string s = "unknown exception caught building HDF5 DAS";
645  throw BESInternalFatalError(s, __FILE__, __LINE__);
646  }
647 
648  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
649  return true;
650 }
651 
652 // Convenient function that helps build DDS and Data
653 // Since this function will be used by both the DDS and Data services, we need to pass both BESDDSResponse and BESDataDDSResponse.
654 // This two parameters are necessary for the future DDS disk cache feature.
655 void HDF5RequestHandler::get_dds_with_attributes( BESDDSResponse*bdds,BESDataDDSResponse*data_bdds,const string &container_name, const string& filename,const string &dds_cache_fname, const string &das_cache_fname,bool dds_from_dc,bool das_from_dc, bool build_data)
656 {
657  DDS *dds;
658  if(true == build_data)
659  dds = data_bdds->get_dds();
660  else dds = bdds->get_dds();
661 
662  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
663  hid_t fileid = -1;
664  hid_t cf_fileid = -1;
665 
666  try {
667 
668  // Look in memory cache to see if it's initialized
669  DDS* cached_dds_ptr = 0;
670  bool use_dds_cache = false;
671  if (dds_cache)
672  cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename));
673  if (cached_dds_ptr)
674  use_dds_cache = true;
675  if (true == use_dds_cache) {
676  // copy the cached DDS into the BES response object. Assume that any cached DDS
677  // includes the DAS information.
678  BESDEBUG(HDF5_NAME, prolog << "DDS Metadata Cached hit for : " << filename << endl);
679  *dds = *cached_dds_ptr; // Copy the referenced object
680  }
681  else if (true ==dds_from_dc) {//Currently the dds_from_ds is always false by default.
682  read_dds_from_disk_cache(bdds,data_bdds,build_data,container_name,filename,dds_cache_fname,das_cache_fname,-1,das_from_dc);
683  }
684  else {
685  BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
686  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
687  dds->filename(filename);
688 
689  // For the time being, not mess up CF's fileID with Default's fileID
690  if(true == _usecf) {
691 
692  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
693  if (cf_fileid < 0){
694  string invalid_file_msg="Could not open this HDF5 file ";
695  invalid_file_msg +=filename;
696  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
697  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
698  invalid_file_msg +=" distributor.";
699  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
700  }
701  // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
702  // to let the handler map the 64-bit integer.
703  if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
704  HDF5RequestHandler::set_dmr_64bit_int(NULL);
705  read_cfdds(*dds,filename,cf_fileid);
706  }
707  else {
708 
709  fileid = get_fileid(filename.c_str());
710  if (fileid < 0) {
711  string invalid_file_msg="Could not open this HDF5 file ";
712  invalid_file_msg +=filename;
713  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
714  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
715  invalid_file_msg +=" distributor.";
716  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
717  }
718 
719  depth_first(fileid, (char*)"/", *dds, filename.c_str());
720 
721  }
722  // Check semantics
723  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
724  dds->print(cerr);
725  throw InternalErr(__FILE__, __LINE__,
726  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
727  }
728 
729  Ancillary::read_ancillary_dds( *dds, filename ) ;
730 
731  // Generate the DDS cached file if needed,currently this is always false by default
732  if(dds_cache_fname!="" && dds_from_dc == false)
733  write_dds_to_disk_cache(dds_cache_fname,dds);
734 
735  // Add attributes
736  {
737  hid_t h5_fd = -1;
738  if(_usecf == true)
739  h5_fd = cf_fileid;
740  else
741  h5_fd = fileid;
742  add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
743  }
744 
745  // Add memory cache if possible
746  if (dds_cache) {
747  // add a copy
748  BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
749  dds_cache->add(new DDS(*dds), filename);
750  }
751 
752  if(cf_fileid != -1)
753  H5Fclose(cf_fileid);
754  if(fileid != -1)
755  H5Fclose(fileid);
756 
757  }
758 
759  }
760  catch(InternalErr & e) {
761 
762  if(cf_fileid !=-1)
763  H5Fclose(cf_fileid);
764 
765  if(fileid != -1)
766  H5Fclose(fileid);
767 
768  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
769  __FILE__, __LINE__);
770  }
771  catch(Error & e) {
772 
773  if(cf_fileid !=-1)
774  H5Fclose(cf_fileid);
775  if(fileid !=-1)
776  H5Fclose(fileid);
777 
778  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
779  __FILE__, __LINE__);
780  }
781  catch(...) {
782 
783  if(cf_fileid !=-1)
784  H5Fclose(cf_fileid);
785  if(fileid !=-1)
786  H5Fclose(fileid);
787 
788  string s = "unknown exception caught building HDF5 DDS";
789  throw BESInternalFatalError(s, __FILE__, __LINE__);
790  }
791 
792 }
793 
794 void HDF5RequestHandler::get_dds_without_attributes_datadds(BESDataDDSResponse*data_bdds,const string &container_name, const string& filename)
795 {
796  DDS *dds = data_bdds->get_dds();
797 
798  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
799  hid_t fileid = -1;
800  hid_t cf_fileid = -1;
801 
802  try {
803 
804  // Look in memory cache to see if it's initialized
805  DDS* cached_dds_ptr = 0;
806  bool use_datadds_cache = false;
807  if (datadds_cache)
808  cached_dds_ptr = static_cast<DDS*>(datadds_cache->get(filename));
809  if (cached_dds_ptr)
810  use_datadds_cache = true;
811  if (true == use_datadds_cache) {
812  // copy the cached DDS into the BES response object.
813  // The DAS information is not included.
814  BESDEBUG(HDF5_NAME, prolog << "DataDDS Metadata Cached hit for : " << filename << endl);
815  *dds = *cached_dds_ptr; // Copy the referenced object
816  }
817  else {
818  BESDEBUG(HDF5_NAME, prolog << "Build DDS from the HDF5 file. " << filename << endl);
819  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
820  dds->filename(filename);
821 
822  // For the time being, not mess up CF's fileID with Default's fileID
823  if(true == _usecf) {
824 
825  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
826  if (cf_fileid < 0){
827  string invalid_file_msg="Could not open this HDF5 file ";
828  invalid_file_msg +=filename;
829  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
830  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
831  invalid_file_msg +=" distributor.";
832  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
833  }
834  // The following is for DAP4 CF(DMR) 64-bit mapping, we need to set the flag
835  // to let the handler map the 64-bit integer.
836  if(HDF5RequestHandler::get_dmr_64bit_int() != NULL)
837  HDF5RequestHandler::set_dmr_64bit_int(NULL);
838  read_cfdds(*dds,filename,cf_fileid);
839  }
840  else {
841 
842  fileid = get_fileid(filename.c_str());
843  if (fileid < 0) {
844  string invalid_file_msg="Could not open this HDF5 file ";
845  invalid_file_msg +=filename;
846  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
847  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
848  invalid_file_msg +=" distributor.";
849  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
850  }
851 
852  depth_first(fileid, (char*)"/", *dds, filename.c_str());
853 
854  }
855  // Check semantics
856  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
857  dds->print(cerr);
858  throw InternalErr(__FILE__, __LINE__,
859  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
860  }
861 
862  Ancillary::read_ancillary_dds( *dds, filename ) ;
863 
864 #if 0
865  // Add attributes
866  {
867  hid_t h5_fd = -1;
868  if(_usecf == true)
869  h5_fd = cf_fileid;
870  else
871  h5_fd = fileid;
872  add_das_to_dds(dds,container_name,filename,das_cache_fname,h5_fd,das_from_dc);
873  }
874 #endif
875 
876  // Add memory cache if possible
877  if (datadds_cache) {
878  // add a copy
879  BESDEBUG(HDF5_NAME, prolog << "DataDDS added to the cache for : " << filename << endl);
880  datadds_cache->add(new DDS(*dds), filename);
881  }
882 
883  if(cf_fileid != -1)
884  H5Fclose(cf_fileid);
885  if(fileid != -1)
886  H5Fclose(fileid);
887 
888  }
889  BESDEBUG(HDF5_NAME, prolog << "Data ACCESS build_data(): set the including attribute flag to false: "<<filename << endl);
890  data_bdds->set_ia_flag(false);
891 
892  }
893  catch(InternalErr & e) {
894 
895  if(cf_fileid !=-1)
896  H5Fclose(cf_fileid);
897 
898  if(fileid != -1)
899  H5Fclose(fileid);
900 
901  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
902  __FILE__, __LINE__);
903  }
904  catch(Error & e) {
905 
906  if(cf_fileid !=-1)
907  H5Fclose(cf_fileid);
908  if(fileid !=-1)
909  H5Fclose(fileid);
910 
911  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
912  __FILE__, __LINE__);
913  }
914  catch(...) {
915 
916  if(cf_fileid !=-1)
917  H5Fclose(cf_fileid);
918  if(fileid !=-1)
919  H5Fclose(fileid);
920 
921  string s = "unknown exception caught building HDF5 DDS";
922  throw BESInternalFatalError(s, __FILE__, __LINE__);
923  }
924 
925 }
926 
927 
928 #if 0
929 // OLD function: Keep it for a while
930 // Convenient function that helps build DDS and Data
931 void HDF5RequestHandler::get_dds_with_attributes(const string &filename, const string &container_name, DDS*dds) {
932 
933  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
934  hid_t fileid = -1;
935  hid_t cf_fileid = -1;
936 
937  try {
938 
939  // Look in memory cache to see if it's initialized
940  DDS* cached_dds_ptr = 0;
941  if (dds_cache && (cached_dds_ptr = static_cast<DDS*>(dds_cache->get(filename)))) {
942  // copy the cached DDS into the BES response object. Assume that any cached DDS
943  // includes the DAS information.
944  BESDEBUG(HDF5_NAME, prolog << "DDS Cached hit for : " << filename << endl);
945  *dds = *cached_dds_ptr; // Copy the referenced object
946  }
947 
948  else {
949 
950  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
951  if (!container_name.empty())
952  dds->container_name(container_name);
953  dds->filename(filename);
954 
955  // For the time being, not mess up CF's fileID with Default's fileID
956  if(true == _usecf) {
957 // This block cannot be used to cache the data
958 #if 0
959  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
960  string dds_filename = "/tmp/"+base_filename+"_dds";
961  FILE *dds_file = fopen(dds_filename.c_str(),"r");
962 cerr<<"before parsing "<<endl;
963 BaseTypeFactory tf;
964 DDS tdds(&tf,name_path(filename),"3.2");
965 tdds.filename(filename);
966  //dds->parse(dds_file);
967  tdds.parse(dds_file);
968  //DDS *cache_dds = new DDS(tdds);
969  cache_dds = new DDS(tdds);
970 if(dds!=NULL)
971  delete dds;
972 dds = cache_dds;
973 tdds.print(cout);
974 dds->print(cout);
975 cerr<<"after parsing "<<endl;
976 //dds->print(cout);
977  // fclose(dds_file);
978 //#endif
979 
980 #endif
981 // end of this block
982 
983 
984  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
985  if (cf_fileid < 0){
986  string invalid_file_msg="Could not open this HDF5 file ";
987  invalid_file_msg +=filename;
988  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
989  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
990  invalid_file_msg +=" distributor.";
991  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
992  }
993 //#if 0
994  read_cfdds(*dds,filename,cf_fileid);
995 //#endif
996  // Generate the DDS cached file
997  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
998  string dds_filename = "/tmp/"+base_filename+"_dds";
999  FILE *dds_file = fopen(dds_filename.c_str(),"w");
1000  dds->print(dds_file);
1001  fclose(dds_file);
1002 
1003  }
1004  else {
1005 
1006  fileid = get_fileid(filename.c_str());
1007  if (fileid < 0) {
1008  string invalid_file_msg="Could not open this HDF5 file ";
1009  invalid_file_msg +=filename;
1010  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1011  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1012  invalid_file_msg +=" distributor.";
1013  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1014  }
1015 
1016  depth_first(fileid, (char*)"/", *dds, filename.c_str());
1017 
1018  }
1019 
1020 
1021  // Check semantics
1022  if (!dds->check_semantics()) { // DDS didn't comply with the semantics
1023  dds->print(cerr);
1024  throw InternalErr(__FILE__, __LINE__,
1025  "DDS check_semantics() failed. This can happen when duplicate variable names are defined. ");
1026  }
1027 
1028  Ancillary::read_ancillary_dds( *dds, filename ) ;
1029 
1030 
1031  // Check DAS cache
1032  DAS *das = 0 ;
1033 
1034  if (das_cache && (das = static_cast<DAS*>(das_cache->get(filename)))) {
1035  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
1036  dds->transfer_attributes(das); // no need to copy the cached DAS
1037  }
1038 
1039  else {
1040 
1041  das = new DAS ;
1042 
1043  if (!container_name.empty())
1044  das->container_name(container_name);
1045 
1046  if (true == _usecf) {
1047 
1048  // go to the CF option
1049  read_cfdas( *das,filename,cf_fileid);
1050 
1051  }
1052  else {
1053 
1054  find_gloattr(fileid, *das);
1055  depth_first(fileid, "/", *das);
1056  close_fileid(fileid);
1057  }
1058 
1059  if(cf_fileid != -1)
1060  H5Fclose(cf_fileid);
1061 
1062  Ancillary::read_ancillary_das( *das, filename ) ;
1063 
1064  dds->transfer_attributes(das);
1065 
1066 
1067  // Only free the DAS if it's not added to the cache
1068  if (das_cache) {
1069  // add a copy
1070  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
1071  //das_cache->add(new DAS(*das), filename);
1072  das_cache->add(das, filename);
1073  }
1074  else {
1075  delete das;
1076  }
1077  }
1078 
1079  if (dds_cache) {
1080  // add a copy
1081  BESDEBUG(HDF5_NAME, prolog << "DDS added to the cache for : " << filename << endl);
1082  dds_cache->add(new DDS(*dds), filename);
1083  }
1084 
1085  }
1086 
1087 //dds->print(cout);
1088 
1089  }
1090  catch(InternalErr & e) {
1091 
1092  if(cf_fileid !=-1)
1093  H5Fclose(cf_fileid);
1094 
1095  if(fileid != -1)
1096  H5Fclose(fileid);
1097 
1098  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1099  __FILE__, __LINE__);
1100  }
1101  catch(Error & e) {
1102 
1103  if(cf_fileid !=-1)
1104  H5Fclose(cf_fileid);
1105  if(fileid !=-1)
1106  H5Fclose(fileid);
1107 
1108  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1109  __FILE__, __LINE__);
1110  }
1111  catch(...) {
1112 
1113  if(cf_fileid !=-1)
1114  H5Fclose(cf_fileid);
1115  if(fileid !=-1)
1116  H5Fclose(fileid);
1117 
1118  string s = "unknown exception caught building HDF5 DDS";
1119  throw BESInternalFatalError(s, __FILE__, __LINE__);
1120  }
1121 
1122 }
1123 #endif
1124 
1125 // Build DDS
1126 bool HDF5RequestHandler::hdf5_build_dds(BESDataHandlerInterface & dhi)
1127 {
1128  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1129 #if DYNAMIC_CONFIG_ENABLED
1130  load_config();
1131 #endif
1132 
1133  // Obtain the HDF5 file name.
1134  string filename = dhi.container->access();
1135 
1136  string container_name = dhi.container->get_symbolic_name();
1137  BESResponseObject *response = dhi.response_handler->get_response_object();
1138  BESDDSResponse *bdds = dynamic_cast < BESDDSResponse * >(response);
1139  if( !bdds )
1140  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1141  bdds->set_container(container_name);
1142 
1143  try {
1144 
1145  bool dds_from_dc = false;
1146  bool das_from_dc = false;
1147  bool build_data = false;
1148  string dds_cache_fname;
1149  string das_cache_fname;
1150 
1151  if(_use_disk_meta_cache) {
1152 
1153  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1154 
1155  // The _use_disk_dds_cache is always set to false by default
1156  if(_use_disk_dds_cache) {
1157  dds_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_dds";
1158  if(access(dds_cache_fname.c_str(),F_OK) !=-1)
1159  dds_from_dc = true;
1160  }
1161 
1162  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1163  // Check if das files exist
1164  if(access(das_cache_fname.c_str(),F_OK) !=-1)
1165  das_from_dc = true;
1166 
1167  }
1168 
1169  get_dds_with_attributes(bdds, NULL,container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1170 
1171  // The following block reads dds from a dds cache file.
1172 #if 0
1173  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1174  string dds_filename = "/tmp/"+base_filename+"_dds";
1175 
1176  BaseTypeFactory tf;
1177  DDS tdds(&tf,name_path(filename),"3.2");
1178  tdds.filename(filename);
1179 
1180 
1181  FILE *dds_file = fopen(dds_filename.c_str(),"r");
1182  tdds.parse(dds_file);
1183 //cerr<<"before parsing "<<endl;
1184  DDS* cache_dds = new DDS(tdds);
1185  if(dds != NULL)
1186  delete dds;
1187  bdds->set_dds(cache_dds);
1188  fclose(dds_file);
1189 #endif
1190 
1191  bdds->set_constraint( dhi ) ;
1192  bdds->clear_container() ;
1193 
1194  }
1195  catch(BESError & e) {
1196  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1197  throw;
1198  }
1199  catch(InternalErr & e) {
1200 
1201  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1202  __FILE__, __LINE__);
1203  }
1204  catch(Error & e) {
1205 
1206  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1207  __FILE__, __LINE__);
1208  }
1209  catch(...) {
1210 
1211  string s = "unknown exception caught building HDF5 DDS";
1212  throw BESInternalFatalError(s, __FILE__, __LINE__);
1213  }
1214 
1215  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1216  return true;
1217 }
1218 
1219 bool HDF5RequestHandler::hdf5_build_data(BESDataHandlerInterface & dhi)
1220 {
1221  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1222 #if DYNAMIC_CONFIG_ENABLED
1223  load_config();
1224 #endif
1225 
1226  if(true ==_usecf) {
1227 
1228  if(true == _pass_fileid)
1229  return hdf5_build_data_with_IDs(dhi);
1230  }
1231 
1232  string filename = dhi.container->access();
1233 
1234  string container_name = dhi.container->get_symbolic_name();
1235  BESResponseObject *response = dhi.response_handler->get_response_object();
1236  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1237  if( !bdds )
1238  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1239  bdds->set_container(container_name);
1240 
1241  try {
1242 
1243  bool dds_from_dc = false;
1244  bool das_from_dc = false;
1245  bool build_data = true;
1246  string dds_cache_fname;
1247  string das_cache_fname;
1248 
1249 
1250  // Only DAS is read from the cache. dds_from_dc is always false.
1251  if(_use_disk_meta_cache == true) {
1252 
1253  string base_filename = HDF5CFUtil::obtain_string_after_lastslash(filename);
1254  das_cache_fname = _disk_meta_cache_path+"/" +base_filename+"_das";
1255 
1256  if(access(das_cache_fname.c_str(),F_OK) !=-1)
1257  das_from_dc = true;
1258 
1259  }
1260 
1261  //get_dds_with_attributes(NULL,bdds, container_name,filename, dds_cache_fname,das_cache_fname,dds_from_dc,das_from_dc,build_data);
1262  get_dds_without_attributes_datadds(bdds,container_name,filename);
1263 
1264  bdds->set_constraint( dhi ) ;
1265  bdds->clear_container() ;
1266 
1267  }
1268  catch(BESError & e) {
1269  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1270  throw;
1271  }
1272  catch(InternalErr & e) {
1273 
1274  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1275  __FILE__, __LINE__);
1276  }
1277  catch(Error & e) {
1278 
1279  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1280  __FILE__, __LINE__);
1281  }
1282  catch(...) {
1283 
1284  string s = "unknown exception caught building HDF5 DDS";
1285  throw BESInternalFatalError(s, __FILE__, __LINE__);
1286  }
1287 
1288  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1289  return true;
1290 }
1291 
1292 // Obtain data when turning on the pass fileID key.The memory cache is not used.
1293 bool HDF5RequestHandler::hdf5_build_data_with_IDs(BESDataHandlerInterface & dhi)
1294 {
1295  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1296 #if DYNAMIC_CONFIG_ENABLED
1297  load_config();
1298 #endif
1299 
1300  BESDEBUG(HDF5_NAME,prolog << "Building DataDDS by passing file IDs. "<<endl);
1301  hid_t cf_fileid = -1;
1302 
1303  string filename = dhi.container->access();
1304 
1305  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1306  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1307  if (cf_fileid < 0){
1308  string invalid_file_msg="Could not open this HDF5 file ";
1309  invalid_file_msg +=filename;
1310  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1311  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1312  invalid_file_msg +=" distributor.";
1313  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1314  }
1315 
1316  BESResponseObject *response = dhi.response_handler->get_response_object();
1317  BESDataDDSResponse *bdds = dynamic_cast < BESDataDDSResponse * >(response);
1318  if( !bdds )
1319  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1320 
1321  try {
1322 
1323  bdds->set_container( dhi.container->get_symbolic_name() ) ;
1324 
1325  HDF5DDS *hdds = new HDF5DDS(bdds->get_dds());
1326  delete bdds->get_dds();
1327 
1328  bdds->set_dds(hdds);
1329 
1330  hdds->setHDF5Dataset(cf_fileid);
1331 
1332  read_cfdds( *hdds,filename,cf_fileid);
1333 
1334  if (!hdds->check_semantics()) { // DDS didn't comply with the DAP semantics
1335  hdds->print(cerr);
1336  throw InternalErr(__FILE__, __LINE__,
1337  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1338  }
1339 
1340  Ancillary::read_ancillary_dds( *hdds, filename ) ;
1341 
1342  DAS *das = new DAS ;
1343  BESDASResponse bdas( das ) ;
1344  bdas.set_container( dhi.container->get_symbolic_name() ) ;
1345  read_cfdas( *das,filename,cf_fileid);
1346  Ancillary::read_ancillary_das( *das, filename ) ;
1347 
1348  hdds->transfer_attributes(das);
1349  bdds->set_constraint( dhi ) ;
1350  bdds->clear_container() ;
1351 
1352  }
1353 
1354  catch(BESError & e) {
1355  if(cf_fileid !=-1)
1356  H5Fclose(cf_fileid);
1357  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1358  throw;
1359  }
1360  catch(InternalErr & e) {
1361  if(cf_fileid !=-1)
1362  H5Fclose(cf_fileid);
1363  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1364  __FILE__, __LINE__);
1365  }
1366  catch(Error & e) {
1367  if(cf_fileid !=-1)
1368  H5Fclose(cf_fileid);
1369  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1370  __FILE__, __LINE__);
1371  }
1372  catch(...) {
1373  if(cf_fileid !=-1)
1374  H5Fclose(cf_fileid);
1375  string s = "unknown exception caught building HDF5 DataDDS";
1376  throw BESInternalFatalError(s, __FILE__, __LINE__);
1377  }
1378 
1379  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1380  return true;
1381 }
1382 
1383 bool HDF5RequestHandler::hdf5_build_dmr(BESDataHandlerInterface & dhi)
1384 {
1385  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1386 #if DYNAMIC_CONFIG_ENABLED
1387  load_config();
1388 #endif
1389 
1390  // Extract the DMR Response object - this holds the DMR used by the
1391  // other parts of the framework.
1392  BESResponseObject *response = dhi.response_handler->get_response_object();
1393  BESDMRResponse &bes_dmr_response = dynamic_cast<BESDMRResponse &>(*response);
1394 
1395  string filename = dhi.container->access();
1396 
1397  DMR *dmr = bes_dmr_response.get_dmr();
1398 
1399  // For the time being, separate CF file ID from the default file ID(mainly for debugging)
1400  hid_t fileid = -1;
1401  hid_t cf_fileid = -1;
1402 
1403  try {
1404 
1405  DMR* cached_dmr_ptr = 0;
1406  if (dmr_cache){
1407  BESDEBUG(HDF5_NAME, prolog << "Checking DMR cache for : " << filename << endl);
1408  cached_dmr_ptr = static_cast<DMR*>(dmr_cache->get(filename));
1409  }
1410 
1411  if (cached_dmr_ptr) {
1412  // copy the cached DMR into the BES response object
1413  BESDEBUG(HDF5_NAME, prolog << "DMR cache hit for : " << filename << endl);
1414  *dmr = *cached_dmr_ptr; // Copy the referenced object
1415  dmr->set_request_xml_base(bes_dmr_response.get_request_xml_base());
1416  }
1417  else {// No cache
1418 
1419  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1420  D4BaseTypeFactory MyD4TypeFactory;
1421  dmr->set_factory(&MyD4TypeFactory);
1422 
1423  if(true ==_usecf) {// CF option
1424 
1425  if(true == _usecfdmr) {
1426  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1427  if (cf_fileid < 0){
1428  string invalid_file_msg="Could not open this HDF5 file ";
1429  invalid_file_msg +=filename;
1430  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1431  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1432  invalid_file_msg +=" distributor.";
1433  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1434  }
1435  read_cfdmr(dmr,filename,cf_fileid);
1436  H5Fclose(cf_fileid);
1437  bes_dmr_response.set_dap4_constraint(dhi);
1438  bes_dmr_response.set_dap4_function(dhi);
1439  dmr->set_factory(0);
1440 
1441  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1442 
1443  return true;
1444  }
1445 
1446  if(true == _pass_fileid)
1447  return hdf5_build_dmr_with_IDs(dhi);
1448 
1449  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1450  if (cf_fileid < 0){
1451  string invalid_file_msg="Could not open this HDF5 file ";
1452  invalid_file_msg +=filename;
1453  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1454  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1455  invalid_file_msg +=" distributor.";
1456  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1457  }
1458 
1459 
1460  BaseTypeFactory factory;
1461  DDS dds(&factory, name_path(filename), "3.2");
1462  dds.filename(filename);
1463 
1464  DAS das;
1465 
1466  // For the CF option dmr response, we need to map 64-bit integer separately
1467  // So set the flag to map 64-bit integer.
1468  HDF5RequestHandler::set_dmr_64bit_int(dmr);
1469  read_cfdds( dds,filename,cf_fileid);
1470  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1471  dds.print(cerr);
1472  throw InternalErr(__FILE__, __LINE__,
1473  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1474  }
1475 
1476  read_cfdas(das,filename,cf_fileid);
1477  Ancillary::read_ancillary_das( das, filename ) ;
1478 
1479  dds.transfer_attributes(&das);
1480 
1482  if(cf_fileid !=-1)
1483  H5Fclose(cf_fileid);
1484 
1485  dmr->build_using_dds(dds);
1486 
1487  }// "if(true == _usecf)"
1488  else {// default option
1489 
1490  // Obtain the HDF5 file ID.
1491  fileid = get_fileid(filename.c_str());
1492  if (fileid < 0) {
1493  string invalid_file_msg="Could not open this HDF5 file ";
1494  invalid_file_msg +=filename;
1495  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1496  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1497  invalid_file_msg +=" distributor.";
1498  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1499  }
1500 
1501  bool use_dimscale = false;
1502  if(true == _default_handle_dimension)
1503  use_dimscale = check_dimscale(fileid);
1504  dmr->set_name(name_path(filename));
1505  dmr->set_filename(name_path(filename));
1506 
1507  // The breadth_first() function builds the variables and attributes and
1508  // loads them into the root group (building child groups as needed).
1509  // jhrg 4/30/20
1510  D4Group* root_grp = dmr->root();
1511  BESDEBUG("h5", "use_dimscale is "<< use_dimscale <<endl);
1512 
1513  // It is possible that a dimension variable has hardlinks. To make it
1514  // right for the netCDF-4 data model and the current DAP4 implementation,
1515  // we need to choose the shortest path of all hardlinks as the dimension path.
1516  // So to avoid iterate all HDF5 objects multiple times, save the found
1517  // hardlinks and search them when necessary. Note we have to search hardlinks from the root.
1518  // KY 2021-11-15
1519  vector<link_info_t> hdf5_hls;
1520  breadth_first(fileid, fileid,(char*)"/",root_grp,filename.c_str(),use_dimscale,hdf5_hls);
1521 #if 0
1522  BESDEBUG("h5", "build_dmr - before obtain dimensions"<< endl);
1523  D4Dimensions *root_dims = root_grp->dims();
1524  for(D4Dimensions::D4DimensionsIter di = root_dims->dim_begin(), de = root_dims->dim_end(); di != de; ++di) {
1525  BESDEBUG("fonc", "transform_dap4() - check dimensions"<< endl);
1526  BESDEBUG("fonc", "transform_dap4() - dim name is: "<<(*di)->name()<<endl);
1527  BESDEBUG("fonc", "transform_dap4() - dim size is: "<<(*di)->size()<<endl);
1528  BESDEBUG("fonc", "transform_dap4() - fully_qualfied_dim name is: "<<(*di)->fully_qualified_name()<<endl);
1529  //cout <<"dim size is: "<<(*di)->size()<<endl;
1530  //cout <<"dim fully_qualified_name is: "<<(*di)->fully_qualified_name()<<endl;
1531  }
1532  BESDEBUG("h5", "build_dmr - after obtain dimensions"<< endl);
1533 #endif
1534 
1535 #if 0
1536  if(true == use_dimscale)
1537  //breadth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str(),true);
1538  breadth_first(fileid,(char*)"/",root_grp,filename.c_str(),true);
1539  else
1540  depth_first(fileid,(char*)"/",root_grp,filename.c_str());
1541  //depth_first(fileid,(char*)"/",*dmr,root_grp,filename.c_str());
1542 #endif
1543 
1544  close_fileid(fileid);
1545 
1546  }// else (default option)
1547 
1548  // If the cache is turned on, add the memory cache.
1549  if (dmr_cache) {
1550  // add a copy
1551  BESDEBUG(HDF5_NAME, prolog << "DMR added to the cache for : " << filename << endl);
1552  dmr_cache->add(new DMR(*dmr), filename);
1553  }
1554  }// else no cache
1555  }// try
1556  catch(BESError & e) {
1557  if(cf_fileid !=-1)
1558  H5Fclose(cf_fileid);
1559  if(fileid !=-1)
1560  H5Fclose(fileid);
1561  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1562  throw;
1563  }
1564  catch(InternalErr & e) {
1565 
1566  if(cf_fileid !=-1)
1567  H5Fclose(cf_fileid);
1568  if(fileid !=-1)
1569  H5Fclose(fileid);
1570 
1571  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1572  __FILE__, __LINE__);
1573  }
1574  catch(Error & e) {
1575 
1576  if(cf_fileid !=-1)
1577  H5Fclose(cf_fileid);
1578  if(fileid !=-1)
1579  H5Fclose(fileid);
1580  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1581  __FILE__, __LINE__);
1582  }
1583  catch(...) {
1584 
1585  if(cf_fileid !=-1)
1586  H5Fclose(cf_fileid);
1587  if(fileid !=-1)
1588  H5Fclose(fileid);
1589  string s = "unknown exception caught building HDF5 DMR";
1590  throw BESInternalFatalError(s, __FILE__, __LINE__);
1591  }
1592 
1593  //dmr->print(cout);
1594 
1595  // Instead of fiddling with the internal storage of the DHI object,
1596  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1597  // methods to set the constraints. But, why? Ans: from Patrick is that
1598  // in the 'container' mode of BES each container can have a different
1599  // CE.
1600  bes_dmr_response.set_dap4_constraint(dhi);
1601  bes_dmr_response.set_dap4_function(dhi);
1602  dmr->set_factory(0);
1603 
1604  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1605  return true;
1606 }
1607 
1608 // This function is only used when EnableCF is true.
1609 bool HDF5RequestHandler::hdf5_build_dmr_with_IDs(BESDataHandlerInterface & dhi)
1610 {
1611  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
1612 #if DYNAMIC_CONFIG_ENABLED
1613  load_config();
1614 #endif
1615 
1616  BESDEBUG("h5","Building DMR with passing file IDs. "<<endl);
1617  string filename = dhi.container->access();
1618  hid_t cf_fileid = -1;
1619 
1620  H5Eset_auto2(H5E_DEFAULT,NULL,NULL);
1621  cf_fileid = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
1622  if (cf_fileid < 0){
1623  string invalid_file_msg="Could not open this HDF5 file ";
1624  invalid_file_msg +=filename;
1625  invalid_file_msg +=". It is very possible that this file is not an HDF5 file ";
1626  invalid_file_msg +=" but with the .h5/.HDF5 suffix. Please check with the data";
1627  invalid_file_msg +=" distributor.";
1628  throw BESInternalError(invalid_file_msg,__FILE__,__LINE__);
1629  }
1630 
1631  BaseTypeFactory factory;
1632  DDS dds(&factory, name_path(filename), "3.2");
1633  dds.filename(filename);
1634 
1635  DAS das;
1636 
1637  try {
1638 
1639 
1640  // This is the CF option
1641  read_cfdds( dds,filename,cf_fileid);
1642 
1643  if (!dds.check_semantics()) { // DDS didn't comply with the DAP semantics
1644  dds.print(cerr);
1645  throw InternalErr(__FILE__, __LINE__,
1646  "DDS check_semantics() failed. This can happen when duplicate variable names are defined.");
1647  }
1648 
1649  Ancillary::read_ancillary_dds( dds, filename ) ;
1650 
1651 
1652  read_cfdas(das,filename,cf_fileid);
1653 
1654  Ancillary::read_ancillary_das( das, filename ) ;
1655 
1656  dds.transfer_attributes(&das);
1657 
1659  //if(cf_fileid !=-1)
1660  // H5Fclose(cf_fileid);
1661 
1662  }
1663  catch(BESError & e) {
1664  if(cf_fileid !=-1)
1665  H5Fclose(cf_fileid);
1666  BESDEBUG(HDF5_NAME, prolog << "Caught BESError! Message: " << e.get_message() << endl);
1667  throw;
1668  }
1669  catch(InternalErr & e) {
1670 
1671  if(cf_fileid !=-1)
1672  H5Fclose(cf_fileid);
1673 
1674  throw BESDapError(e.get_error_message(), true, e.get_error_code(),
1675  __FILE__, __LINE__);
1676  }
1677  catch(Error & e) {
1678 
1679  if(cf_fileid !=-1)
1680  H5Fclose(cf_fileid);
1681 
1682  throw BESDapError(e.get_error_message(), false, e.get_error_code(),
1683  __FILE__, __LINE__);
1684  }
1685  catch(...) {
1686 
1687  if(cf_fileid !=-1)
1688  H5Fclose(cf_fileid);
1689 
1690  string s = "unknown exception caught building HDF5 DataDDS";
1691  throw BESInternalFatalError(s, __FILE__, __LINE__);
1692  }
1693 
1694  // Extract the DMR Response object - this holds the DMR used by the
1695  // other parts of the framework.
1696  BESResponseObject *response = dhi.response_handler->get_response_object();
1697  BESDMRResponse &bes_dmr = dynamic_cast<BESDMRResponse &>(*response);
1698 
1699  // In this handler we use a different pattern since the handler specializes the DDS/DMR.
1700  // First, build the DMR adding the open handle to the HDF4 dataset, then free the DMR
1701  // the BES built and add this one. The HDF4DMR object will close the open dataset when
1702  // the BES runs the DMR's destructor.
1703 
1704  DMR *dmr = bes_dmr.get_dmr();
1705  D4BaseTypeFactory MyD4TypeFactory;
1706  dmr->set_factory(&MyD4TypeFactory);
1707  dmr->build_using_dds(dds);
1708 
1709  HDF5DMR *hdf5_dmr = new HDF5DMR(dmr);
1710  hdf5_dmr->setHDF5Dataset(cf_fileid);
1711  delete dmr; // The call below will make 'dmr' unreachable; delete it now to avoid a leak.
1712  bes_dmr.set_dmr(hdf5_dmr); // BESDMRResponse will delete hdf5_dmr
1713 
1714  // Instead of fiddling with the internal storage of the DHI object,
1715  // (by setting dhi.data[DAP4_CONSTRAINT], etc., directly) use these
1716  // methods to set the constraints. But, why? Ans: from Patrick is that
1717  // in the 'container' mode of BES each container can have a different
1718  // CE.
1719  bes_dmr.set_dap4_constraint(dhi);
1720  bes_dmr.set_dap4_function(dhi);
1721  hdf5_dmr->set_factory(0);
1722 
1723  BESDEBUG(HDF5_NAME, prolog << "END" << endl);
1724  return true;
1725 }
1726 
1727 bool HDF5RequestHandler::hdf5_build_help(BESDataHandlerInterface & dhi)
1728 {
1729  BESResponseObject *response = dhi.response_handler->get_response_object();
1730  BESInfo *info = dynamic_cast<BESInfo *>(response);
1731  if( !info )
1732  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1733 
1734  string add_info="Just for Test";
1735 
1736  map<string,string> attrs ;
1737  attrs["name"] = MODULE_NAME ;
1738  attrs["version"] = MODULE_VERSION ;
1739  list<string> services ;
1740  BESServiceRegistry::TheRegistry()->services_handled( HDF5_NAME, services );
1741  if( services.size() > 0 )
1742  {
1743  string handles = BESUtil::implode( services, ',' ) ;
1744  attrs["handles"] = handles ;
1745  }
1746  info->begin_tag( "module", &attrs ) ;
1747  info->end_tag( "module" ) ;
1748  info->add_data(add_info);
1749 
1750  return true;
1751 }
1752 
1753 bool HDF5RequestHandler::hdf5_build_version(BESDataHandlerInterface & dhi)
1754 {
1755  BESResponseObject *response = dhi.response_handler->get_response_object();
1756  BESVersionInfo *info = dynamic_cast < BESVersionInfo * >(response);
1757  if( !info )
1758  throw BESInternalError( "cast error", __FILE__, __LINE__ ) ;
1759 
1760  info->add_module( MODULE_NAME, MODULE_VERSION ) ;
1761 
1762  return true;
1763 }
1764 
1765 
1766 bool HDF5RequestHandler::obtain_lrd_common_cache_dirs()
1767 {
1768  string lrd_config_fpath;
1769  string lrd_config_fname;
1770 
1771  // Obtain DataCache path
1772  lrd_config_fpath = get_beskeys("H5.DataCachePath");
1773 
1774  // Obtain the configure file name that specifics the large file configuration
1775  lrd_config_fname = get_beskeys("H5.LargeDataMemCacheFileName");
1776 
1777  // If either the configure file path or fname is missing, won't add specific mem. cache dirs.
1778  if(lrd_config_fpath=="" || lrd_config_fname=="")
1779  return false;
1780 
1781  // temp_line for storing info of one line in the config. file
1782  string temp_line;
1783 
1784  // The full path of the configure file
1785  string mcache_config_fname = lrd_config_fpath+"/"+lrd_config_fname;
1786 
1787  //ifstream mcache_config_file("example.txt");
1788  // Open the configure file
1789  ifstream mcache_config_file(mcache_config_fname.c_str());
1790 
1791  // If the configuration file is not open, return false.
1792  if(mcache_config_file.is_open()==false){
1793  BESDEBUG(HDF5_NAME, prolog << "The large data memory cache configure file "<<mcache_config_fname );
1794  BESDEBUG(HDF5_NAME, prolog << " cannot be opened."<<endl);
1795  return false;
1796  }
1797 
1798  // Read the configuration file line by line
1799  while(getline(mcache_config_file,temp_line)) {
1800 
1801  // Only consider lines that is no less than 2 characters and the 2nd character is space.
1802  if(temp_line.size()>1 && temp_line.at(1)==' ') {
1803  char sep=' ';
1804  string subline = temp_line.substr(2);
1805  vector<string> temp_name_list;
1806 
1807  // Include directories to store common latitude and longitude values
1808  if(temp_line.at(0)=='1') {
1809  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1810  //lrd_cache_dir_list +=temp_name_list;
1811  lrd_cache_dir_list.insert(lrd_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1812  }
1813  // Include directories not to store common latitude and longitude values
1814  else if(temp_line.at(0)=='0'){
1815  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1816  //lrd_non_cache_dir_list +=temp_name_list;
1817  lrd_non_cache_dir_list.insert(lrd_non_cache_dir_list.end(),temp_name_list.begin(),temp_name_list.end());
1818  }
1819  // Include variable names that the server would like to store in the memory cache
1820  else if(temp_line.at(0)=='2') {
1821 
1822  // We need to handle the space case inside a variable path
1823  // either "" or '' needs to be used to identify a var path
1824  vector<int>dq_pos;
1825  vector<int>sq_pos;
1826  for(unsigned int i = 0; i<subline.size();i++){
1827  if(subline[i]=='"') {
1828  dq_pos.push_back(i);
1829  }
1830  else if(subline[i]=='\'')
1831  sq_pos.push_back(i);
1832  }
1833  if(dq_pos.size()==0 && sq_pos.size()==0)
1834  HDF5CFUtil::Split_helper(temp_name_list,subline,sep);
1835  else if((dq_pos.size()!=0) &&(dq_pos.size()%2==0)&& sq_pos.size()==0) {
1836  unsigned int dq_index= 0;
1837  while(dq_index < dq_pos.size()){
1838  if(dq_pos[dq_index+1]>(dq_pos[dq_index]+1)) {
1839  temp_name_list.push_back
1840  (subline.substr(dq_pos[dq_index]+1,dq_pos[dq_index+1]-dq_pos[dq_index]-1));
1841  }
1842  dq_index = dq_index + 2;
1843  }
1844  }
1845  else if((sq_pos.size()!=0) &&(sq_pos.size()%2==0)&& dq_pos.size()==0) {
1846  unsigned int sq_index= 0;
1847  while(sq_index < sq_pos.size()){
1848  if(sq_pos[sq_index+1]>(sq_pos[sq_index]+1)) {
1849  temp_name_list.push_back
1850  (subline.substr(sq_pos[sq_index]+1,sq_pos[sq_index+1]-sq_pos[sq_index]-1));
1851  }
1852  sq_index = sq_index+2;
1853  }
1854  }
1855 
1856  lrd_var_cache_file_list.insert(lrd_var_cache_file_list.end(),temp_name_list.begin(),temp_name_list.end());
1857  }
1858  }
1859  }
1860 
1861 
1862 #if 0
1863 
1864 for(int i =0; i<lrd_cache_dir_list.size();i++)
1865 cerr<<"lrd cache list is "<<lrd_cache_dir_list[i] <<endl;
1866 for(int i =0; i<lrd_non_cache_dir_list.size();i++)
1867 cerr<<"lrd non cache list is "<<lrd_non_cache_dir_list[i] <<endl;
1868 for(int i =0; i<lrd_var_cache_file_list.size();i++)
1869 cerr<<"lrd var cache file list is "<<lrd_var_cache_file_list[i] <<endl;
1870 #endif
1871 
1872 
1873  mcache_config_file.close();
1874  if(lrd_cache_dir_list.size()==0 && lrd_non_cache_dir_list.size()==0 && lrd_var_cache_file_list.size()==0)
1875  return false;
1876  else
1877  return true;
1878 }
1879 
1880 
1881 bool HDF5RequestHandler::read_das_from_disk_cache(const string & cache_filename,DAS *das_ptr) {
1882 
1883  BESDEBUG(HDF5_NAME, prolog << "Coming to read_das_from_disk_cache() " << cache_filename << endl);
1884  bool ret_value = true;
1885  FILE *md_file = NULL;
1886  md_file = fopen(cache_filename.c_str(),"rb");
1887 
1888  if(NULL == md_file) {
1889  string bes_error = "An error occurred trying to open a metadata cache file " + cache_filename;
1890  throw BESInternalError( bes_error, __FILE__, __LINE__);
1891  }
1892  else {
1893 
1894  int fd_md = fileno(md_file);
1895  struct flock *l_md;
1896  l_md = lock(F_RDLCK);
1897 
1898  // hold a read(shared) lock to read metadata from a file.
1899  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1900  fclose(md_file);
1901  ostringstream oss;
1902  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1903  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1904  }
1905 
1906  try {
1907 
1908  struct stat sb;
1909  if(stat(cache_filename.c_str(),&sb) != 0) {
1910  string bes_error = "An error occurred trying to stat a metadata cache file size " + cache_filename;
1911  throw BESInternalError( bes_error, __FILE__, __LINE__);
1912 
1913  }
1914 
1915 
1916  size_t bytes_expected_read=(size_t)sb.st_size;
1917  BESDEBUG(HDF5_NAME, prolog << "DAS Disk cache file size is " << bytes_expected_read << endl);
1918 
1919  vector<char> buf;
1920  buf.resize(bytes_expected_read);
1921  size_t bytes_to_read =fread((void*)&buf[0],1,bytes_expected_read,md_file);
1922  if(bytes_to_read != bytes_expected_read)
1923  throw InternalErr(__FILE__,__LINE__,"Fail to read the data from the das cache file.");
1924 
1925  char* temp_pointer =&buf[0];
1926 
1927  AttrTable*at = NULL;
1928  // recursively build DAS
1929 //#if 0
1930  temp_pointer = get_attr_info_from_dc(temp_pointer,das_ptr,at);
1931 //#endif
1932 
1933 
1934  }
1935  catch(...) {
1936  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1937  fclose(md_file);
1938  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1939  }
1940 
1941  fclose(md_file);
1942  throw InternalErr(__FILE__,__LINE__,"Fail to parse a das cache file.");
1943  }
1944 
1945  // Unlock the cache file
1946  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1947  fclose(md_file);
1948  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1949  }
1950  fclose(md_file);
1951  }
1952  return ret_value;
1953 
1954 }
1955 
1956 // This fucntion will NOT be used by default. Leave here for future improvement.
1957 bool HDF5RequestHandler::write_dds_to_disk_cache(const string& dds_cache_fname,DDS *dds_ptr) {
1958 
1959  BESDEBUG(HDF5_NAME, prolog << "Write DDS to disk cache " << dds_cache_fname << endl);
1960  FILE *dds_file = fopen(dds_cache_fname.c_str(),"w");
1961 
1962  if(NULL == dds_file) {
1963  string bes_error = "An error occurred trying to open a metadata cache file " + dds_cache_fname;
1964  throw BESInternalError( bes_error, __FILE__, __LINE__);
1965  }
1966  else {
1967 
1968  int fd_md = fileno(dds_file);
1969  struct flock *l_md;
1970  l_md = lock(F_WRLCK);
1971 
1972  // hold a read(shared) lock to read metadata from a file.
1973  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
1974  fclose(dds_file);
1975  ostringstream oss;
1976  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
1977  throw BESInternalError( oss.str(), __FILE__, __LINE__);
1978  }
1979 
1980  try {
1981  dds_ptr->print(dds_file);
1982  }
1983  catch(...) {
1984  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1985  fclose(dds_file);
1986  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1987  }
1988 
1989  fclose(dds_file);
1990  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
1991  }
1992 
1993  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
1994  fclose(dds_file);
1995  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
1996  }
1997 
1998  fclose(dds_file);
1999  }
2000  return true;
2001 
2002 }
2003 
2004 // Write DAS to a binary cached file on disk.
2005 bool HDF5RequestHandler::write_das_to_disk_cache(const string & das_cache_fname, DAS *das_ptr) {
2006 
2007  BESDEBUG(HDF5_NAME, prolog << "Write DAS to disk cache " << das_cache_fname << endl);
2008  FILE *das_file = fopen(das_cache_fname.c_str(),"wb");
2009  if(NULL == das_file) {
2010  string bes_error = "An error occurred trying to open a metadata cache file " + das_cache_fname;
2011  throw BESInternalError( bes_error, __FILE__, __LINE__);
2012  }
2013  else {
2014  int fd_md = fileno(das_file);
2015  struct flock *l_md;
2016  l_md = lock(F_WRLCK);
2017 
2018  // hold a write(exclusive) lock to write metadata to a file.
2019  if(fcntl(fd_md,F_SETLKW,l_md) == -1) {
2020  fclose(das_file);
2021  ostringstream oss;
2022  oss << "cache process: " << l_md->l_pid << " triggered a locking error: " << get_errno();
2023  throw BESInternalError( oss.str(), __FILE__, __LINE__);
2024  }
2025 
2026  try {
2027  write_das_to_file(das_ptr,das_file);
2028  }
2029  catch(...) {
2030  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2031  fclose(das_file);
2032  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2033  }
2034 
2035  fclose(das_file);
2036  throw InternalErr(__FILE__,__LINE__,"Fail to parse a dds cache file.");
2037  }
2038 
2039  if(fcntl(fd_md,F_SETLK,lock(F_UNLCK)) == -1) {
2040  fclose(das_file);
2041  throw BESInternalError( "An error occurred trying to unlock the file" + get_errno(), __FILE__, __LINE__);
2042  }
2043 
2044  fclose(das_file);
2045 
2046  }
2047 
2048  return true;
2049 
2050 }
2051 
2052 // The wrapper function to call write_das_table_to_file to generate the cache.
2053 void write_das_to_file(DAS*das_ptr,FILE* das_file) {
2054 
2055  // When category_flag is 2, it marks the end of the file.
2056  uint8_t category_flag = 2;
2057  AttrTable* top_table = das_ptr->get_top_level_attributes();
2058  write_das_table_to_file(top_table,das_file);
2059 
2060  // Add the final ending flag for retrieving the info.
2061  fwrite((const void*)&category_flag,1,1,das_file);
2062  return;
2063 
2064 }
2065 
2066 // The main function to write DAS to a file
2067 void write_das_table_to_file(AttrTable*temp_table,FILE* das_file) {
2068 
2069  if(temp_table !=NULL) {
2070 
2071  // 2 is the end mark of an attribute table
2072  uint8_t category_flag = 2;
2073 
2074  // Loop through the whole DAS top table
2075  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2076  AttrTable::Attr_iter top_endit = temp_table->attr_end();
2077  AttrTable::Attr_iter top_it = top_startit;
2078  while(top_it !=top_endit) {
2079  AttrType atype = temp_table->get_attr_type(top_it);
2080  if(atype == Attr_unknown)
2081  throw InternalErr(__FILE__,__LINE__,"Unsupported DAS Attribute type");
2082  else if(atype!=Attr_container) {
2083  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr name is: "
2084  << temp_table->get_name(top_it) << endl);
2085  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr type is: "
2086  << temp_table->get_type(top_it) << endl);
2087  // For the debugging purpose
2088  //unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2089  //cerr<<"Attribute values are "<<endl;
2090  //for (int i = 0; i <num_attrs;i++)
2091  // cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2092  //cerr<<endl;
2093  //write_das_attr_info(temp_table,top_it,das_file);
2094  // Write DAS attribute info to the file
2095  write_das_attr_info(temp_table,temp_table->get_name(top_it),temp_table->get_type(top_it),das_file);
2096  }
2097  else {
2098  BESDEBUG(HDF5_NAME, prolog << "DAS to the disk cache, attr container name is: "
2099  << (*top_it)->name << endl);
2100  // Write the container and then write the info. in this container
2101  AttrTable* sub_table = temp_table->get_attr_table(top_it);
2102  write_container_name_to_file(sub_table->get_name(),das_file);
2103  write_das_table_to_file(sub_table,das_file);
2104 
2105  // Write the end flag
2106  fwrite((const void*)&category_flag,1,1,das_file);
2107 
2108  }
2109  ++top_it;
2110  }
2111 
2112  }
2113 }
2114 
2115 // Write container name to the disk file
2116 void write_container_name_to_file(const string& cont_name,FILE *das_file) {
2117 
2118  // 1 marks the starting of a container
2119  uint8_t category_flag = 1;
2120  vector<char> buf;
2121  size_t bytes_to_write = cont_name.size()+sizeof(size_t)+1;
2122  buf.resize(bytes_to_write);
2123  char*temp_pointer =&buf[0];
2124  memcpy((void*)temp_pointer,(void*)&category_flag,1);
2125  temp_pointer++;
2126  temp_pointer=copy_str(temp_pointer,cont_name);
2127 
2128  size_t bytes_to_be_written = fwrite((const void*)&buf[0],1,bytes_to_write,das_file);
2129  if(bytes_to_be_written != bytes_to_write)
2130  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS container name to a cache");
2131  return;
2132 }
2133 
2134 
2135 // Write DAS attribute info. to the disk cache file
2136 void write_das_attr_info(AttrTable* dtp,const string& attr_name, const string & attr_type,FILE * das_file) {
2137 
2138  // 0 marks the starting of a DAS attribute
2139  uint8_t category_flag = 0;
2140 
2141  unsigned int num_attr_elems = dtp->get_attr_num(attr_name);
2142  vector<string> attr_values;
2143  size_t total_attr_values_size = 0;
2144  for (unsigned int i = 0; i <num_attr_elems;i++){
2145  attr_values.push_back((*(dtp->get_attr_vector(attr_name)))[i]);
2146  total_attr_values_size += attr_values[i].size();
2147  }
2148  // Need to add a flag, value as 0 to indicate the attribute.
2149  // DAS: category flag, sizeof attirubte name, attribute name, size of attribute type, attribute type
2150  size_t bytes_to_write_attr = 1 + attr_name.size() + attr_type.size() + 2* sizeof(size_t);
2151 
2152  // One unsigned int to store the number of element elements i
2153  // + sizeof(size_t) * number of elements to store the number of characters for each attribute
2154  // (in DAP, every attribute is in string format)
2155  // +total size of all attribute values
2156  bytes_to_write_attr += sizeof(unsigned int) + num_attr_elems*sizeof(size_t)+total_attr_values_size;
2157 
2158  vector<char>attr_buf;
2159  attr_buf.resize(bytes_to_write_attr);
2160  char* temp_attrp =&attr_buf[0];
2161 
2162  // The attribute flag
2163  memcpy((void*)temp_attrp,(void*)&category_flag,1);
2164  temp_attrp++;
2165 
2166  // The attribute name and type
2167  temp_attrp=copy_str(temp_attrp,attr_name);
2168  temp_attrp=copy_str(temp_attrp,attr_type);
2169 
2170  // Number of elements
2171  memcpy((void*)temp_attrp,(void*)&num_attr_elems,sizeof(unsigned int));
2172  temp_attrp+=sizeof(unsigned int);
2173 
2174  // All attributes
2175  for (unsigned int i = 0; i <num_attr_elems;i++)
2176  temp_attrp=copy_str(temp_attrp,(*(dtp->get_attr_vector(attr_name)))[i]);
2177 
2178  size_t bytes_to_be_written = fwrite((const void*)&attr_buf[0],1,bytes_to_write_attr,das_file);
2179  if(bytes_to_be_written != bytes_to_write_attr)
2180  throw InternalErr(__FILE__, __LINE__,"Failed to write a DAS attribute to a cache");
2181 
2182  return;
2183 
2184 }
2185 
2186 // Read DDS from a disk cache, this function is not used by default.
2187 void HDF5RequestHandler::read_dds_from_disk_cache(BESDDSResponse* bdds, BESDataDDSResponse* data_bdds,
2188  bool build_data,const string & container_name,const string & h5_fname,
2189  const string & dds_cache_fname,const string &das_cache_fname, hid_t h5_fd,
2190  bool das_from_dc) {
2191 
2192 
2193  BESDEBUG(HDF5_NAME, prolog << "BEGIN dds_cache_fname: " << dds_cache_fname << endl);
2194 
2195  DDS *dds;
2196  if(true == build_data)
2197  dds = data_bdds->get_dds();
2198  else
2199  dds = bdds->get_dds();
2200 
2201  // write a function to pass the following with the lock.
2202  BaseTypeFactory tf;
2203  DDS tdds(&tf,name_path(h5_fname),"3.2");
2204  tdds.filename(h5_fname);
2205 
2206  FILE *dds_file = fopen(dds_cache_fname.c_str(),"r");
2207  tdds.parse(dds_file);
2208  DDS* cache_dds = new DDS(tdds);
2209 #if 0
2210 cerr<<"before dds "<<endl;
2211 dds->dump(cerr);
2212 cerr<<"after dds "<<endl;
2213 cerr<<"before tdds "<<endl;
2214 cache_dds->dump(cerr);
2215 cerr<<"after tdds "<<endl;
2216 #endif
2217  if(dds != NULL)
2218  delete dds;
2219 
2220  Ancillary::read_ancillary_dds( *cache_dds, h5_fname ) ;
2221 
2222  add_das_to_dds(cache_dds,container_name,h5_fname,das_cache_fname,h5_fd,das_from_dc);
2223  if(true == build_data)
2224  data_bdds->set_dds(cache_dds);
2225  else
2226  bdds->set_dds(cache_dds);
2227  fclose(dds_file);
2228 
2229  if (dds_cache) {
2230  // add a copy
2231  BESDEBUG(HDF5_NAME, prolog << "For memory cache, DDS added to the cache for : " << h5_fname << endl);
2232  dds_cache->add(new DDS(*cache_dds), h5_fname);
2233  }
2234 
2235 }
2236 
2237 // Add DAS to DDS.
2238 void HDF5RequestHandler::add_das_to_dds(DDS *dds, const string &/*container_name*/, const string &filename,
2239  const string &das_cache_fname, hid_t h5_fd, bool das_from_dc) {
2240 
2241  BESDEBUG(HDF5_NAME, prolog << "BEGIN" << endl);
2242 
2243  // Check DAS memory cache
2244  DAS *das = 0 ;
2245  bool use_das_cache = false;
2246  if (das_cache)
2247  das = static_cast<DAS*>(das_cache->get(filename));
2248  if (das)
2249  use_das_cache = true;
2250 
2251  if (true == use_das_cache) {
2252  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2253  dds->transfer_attributes(das); // no need to copy the cached DAS
2254  }
2255 
2256  else {
2257 
2258  das = new DAS ;
2259 #if 0
2260  if (!container_name.empty())
2261  das->container_name(container_name);
2262 #endif
2263  if(das_from_dc == true)
2264  read_das_from_disk_cache(das_cache_fname,das);
2265  else {
2266  // This bool is for the case, when DDS is read from a cache then we need to open the HDF5 file.
2267  bool h5_file_open = true;
2268  if(h5_fd == -1)
2269  h5_file_open = false;
2270  if (true == _usecf) {
2271  // go to the CF option
2272  if(h5_file_open == false)
2273  h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2274 
2275  read_cfdas( *das,filename,h5_fd);
2276  if(h5_file_open == false)
2277  H5Fclose(h5_fd);
2278  }
2279  else {
2280  if(h5_file_open == false)
2281  h5_fd = get_fileid(filename.c_str());
2282  find_gloattr(h5_fd, *das);
2283  depth_first(h5_fd, "/", *das);
2284  if(h5_file_open == false)
2285  close_fileid(h5_fd);
2286  }
2287 
2288  Ancillary::read_ancillary_das( *das, filename ) ;
2289 
2290  if(das_cache_fname!="" && das_from_dc == false)
2291  write_das_to_disk_cache(das_cache_fname,das);
2292  }
2293 
2294  dds->transfer_attributes(das);
2295 
2296  if (das_cache) {
2297  // add a copy
2298  BESDEBUG(HDF5_NAME, prolog << "For memory cache, DAS added to the cache for : " << filename << endl);
2299  das_cache->add(new DAS(*das), filename);
2300  }
2301  delete das;
2302 
2303  }
2304 
2305 }
2306 
2307 bool obtain_beskeys_info(const string& key, bool & has_key) {
2308 
2309  bool ret_value = false;
2310  string doset ="";
2311  TheBESKeys::TheKeys()->get_value( key, doset, has_key ) ;
2312  if(has_key) {
2313  const string dosettrue ="true";
2314  const string dosetyes = "yes";
2315  doset = BESUtil::lowercase(doset) ;
2316  ret_value = (dosettrue == doset || dosetyes == doset);
2317  }
2318  return ret_value;
2319 }
2320 
2321 #if 0
2322 bool is_beskeys_set_true(const string& bes_value) {
2323 
2324  const string dosettrue ="true";
2325  const string dosetyes = "yes";
2326  string doset = BESUtil::lowercase(bes_value) ;
2327  return (dosettrue == doset || dosetyes == doset);
2328 
2329 }
2330 bool check_and_set_beskeys(const string key) {
2331 
2332  bool found = false;
2333  string doset ="";
2334  const string dosettrue ="true";
2335  const string dosetyes = "yes";
2336 
2337  TheBESKeys::TheKeys()->get_value( key, doset, found ) ;
2338  if( true == found ) {
2339  doset = BESUtil::lowercase( doset ) ;
2340  }
2341  BESDEBUG(HDF5_NAME, prolog << "Key: " << key << (found?(" was found. value: "+doset):" was not found.") << endl);
2342  return found && (dosettrue == doset || dosetyes == doset);
2343 
2344 }
2345 #endif
2346 
2347 // get_uint_key and get_float_key are copied from the netCDF handler.
2348 
2349 static unsigned int get_uint_key(const string &key, unsigned int def_val)
2350 {
2351  bool found = false;
2352  string doset = "";
2353 
2354  TheBESKeys::TheKeys()->get_value(key, doset, found);
2355  if (true == found) {
2356  // In C++11, stoi is better.
2357  return atoi(doset.c_str()); // use better code TODO
2358  }
2359  else {
2360  return def_val;
2361  }
2362 }
2363 
2364 static unsigned long get_ulong_key(const string &key, unsigned long def_val)
2365 {
2366  bool found = false;
2367  string doset = "";
2368 
2369  TheBESKeys::TheKeys()->get_value(key, doset, found);
2370  if (true == found) {
2371  // In C++11, stoull is better.
2372  return atol(doset.c_str()); // use better code TODO
2373  }
2374  else {
2375  return def_val;
2376  }
2377 }
2378 static float get_float_key(const string &key, float def_val)
2379 {
2380  bool found = false;
2381  string doset = "";
2382 
2383  TheBESKeys::TheKeys()->get_value(key, doset, found);
2384  if (true == found) {
2385  return atof(doset.c_str()); // use better code TODO
2386  }
2387  else {
2388  return def_val;
2389  }
2390 }
2391 
2392 static string get_beskeys(const string &key) {
2393 
2394  bool found = false;
2395  string ret_value ="";
2396 
2397  TheBESKeys::TheKeys()->get_value( key, ret_value, found ) ;
2398  return ret_value;
2399 
2400 }
2401 
2402 // The function to copy a string to a memory buffer.
2403 char* copy_str(char*temp_ptr,const string & str) {
2404 
2405  size_t str_size=str.size();
2406  memcpy((void*)temp_ptr,(void*)&str_size,sizeof(size_t));
2407  temp_ptr+=sizeof(size_t);
2408  vector<char>temp_vc2(str.begin(),str.end());
2409  memcpy((void*)temp_ptr,(void*)&temp_vc2[0],str.size());
2410  temp_ptr+=str.size();
2411  return temp_ptr;
2412 
2413 }
2414 
2415 
2416 // Obtain the string from a memory buffer.
2417 // Note: both char* and string(as a reference) will be returned
2418 // The attribute binary first stores the size of the string, then the string itself
2419 char* obtain_str(char*temp_ptr,string & str) {
2420 
2421  size_t oname_size = *((size_t *)temp_ptr);
2422  temp_ptr = temp_ptr + sizeof(size_t);
2423  string oname;
2424  for(unsigned int i =0; i<oname_size; i++){
2425  oname.push_back(*temp_ptr);
2426  ++temp_ptr;
2427  }
2428  str = oname;
2429  return temp_ptr;
2430 
2431 }
2432 
2433 // For our case, there are no global attributes for DAS.
2434 // The global attribures are always under HDF_GLOBAL.
2435 // The main function to obtain the DAS info. from the cache.
2436 char* get_attr_info_from_dc(char*temp_pointer,DAS *das,AttrTable *at_par) {
2437 
2438  // 3 is only for the code to come into the loop.
2439  uint8_t flag =3;
2440  while(flag !=2) {
2441  flag = *((uint8_t*)(temp_pointer));
2442  BESDEBUG(HDF5_NAME, prolog << "Build DAS from the disk cache file flag: "
2443  <<" flag = 0, attribute; flag = 1, container; flag =2; end of container;"
2444  <<" flag = 3; the initial value to get the attribute retrieval process started."
2445  <<" The flag value is "
2446  << (int)flag <<endl);
2447  temp_pointer++;
2448 
2449  if(flag ==1) {
2450  string container_name;
2451  temp_pointer = obtain_str(temp_pointer,container_name);
2452  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, container name is " << container_name << endl);
2453 
2454  // Remember the current Attribute table state
2455  AttrTable*temp_at_par = at_par;
2456  if(at_par == NULL)
2457  at_par = das->add_table(container_name, new AttrTable);
2458  else
2459  at_par = at_par->append_container(container_name);
2460 
2461  temp_pointer = get_attr_info_from_dc(temp_pointer,das,at_par);
2462  // MUST resume the original state
2463  at_par = temp_at_par;
2464 
2465  }
2466  else if(flag == 0) {
2467  // The attribute must have a table.
2468  if(at_par ==NULL)
2469  throw BESInternalError( "The AttrTable must exist for DAS attributes", __FILE__, __LINE__ ) ;
2470 
2471  // Attribute name
2472  string attr_name;
2473  temp_pointer = obtain_str(temp_pointer,attr_name);
2474  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr name is: " << attr_name << endl);
2475 
2476  // Attribute type
2477  string attr_type;
2478  temp_pointer = obtain_str(temp_pointer,attr_type);
2479  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attr type is: " << attr_type << endl);
2480 
2481  // Attribute values
2482  unsigned int num_values = *((unsigned int*)(temp_pointer));
2483  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, number of attribute values is: " << num_values << endl);
2484  temp_pointer+=sizeof(unsigned int);
2485 
2486  vector <string> attr_values;
2487 
2488  for(unsigned int i = 0; i<num_values; i++) {
2489  string attr_value;
2490  temp_pointer = obtain_str(temp_pointer,attr_value);
2491  attr_values.push_back(attr_value);
2492  BESDEBUG(HDF5_NAME, prolog << "DAS from the disk cache, attribute value is: " << attr_value << endl);
2493  }
2494 
2495  at_par->append_attr(attr_name,attr_type,&attr_values);
2496  }
2497 
2498  }
2499  return temp_pointer;
2500 
2501 }
2502 
2503 // The debugging function to get attribute info.
2504 void get_attr_contents(AttrTable*temp_table) {
2505  if(temp_table !=NULL) {
2506  AttrTable::Attr_iter top_startit = temp_table->attr_begin();
2507  AttrTable::Attr_iter top_endit = temp_table->attr_end();
2508  AttrTable::Attr_iter top_it = top_startit;
2509  while(top_it !=top_endit) {
2510  AttrType atype = temp_table->get_attr_type(top_it);
2511  if(atype == Attr_unknown)
2512  cerr<<"unsupported DAS attributes" <<endl;
2513  else if(atype!=Attr_container) {
2514 
2515  cerr<<"Attribute name is "<<temp_table->get_name(top_it)<<endl;
2516  cerr<<"Attribute type is "<<temp_table->get_type(top_it)<<endl;
2517  unsigned int num_attrs = temp_table->get_attr_num(temp_table->get_name(top_it));
2518  cerr<<"Attribute values are "<<endl;
2519  for (unsigned int i = 0; i <num_attrs;i++)
2520  cerr<<(*(temp_table->get_attr_vector(temp_table->get_name(top_it))))[i]<<" ";
2521  cerr<<endl;
2522  }
2523  else {
2524  cerr<<"Coming to the attribute container. "<<endl;
2525  cerr<<"container name is "<<(*top_it)->name <<endl;
2526  AttrTable* sub_table = temp_table->get_attr_table(top_it);
2527  cerr<<"container table name is "<<sub_table->get_name() <<endl;
2528  get_attr_contents(sub_table);
2529  }
2530  ++top_it;
2531  }
2532 
2533  }
2534 }
2535 
2536 
2537 void HDF5RequestHandler::add_attributes(BESDataHandlerInterface &dhi) {
2538 
2539 
2540  BESResponseObject *response = dhi.response_handler->get_response_object();
2541  BESDataDDSResponse *bdds = dynamic_cast<BESDataDDSResponse *>(response);
2542  if (!bdds)
2543  throw BESInternalError("cast error", __FILE__, __LINE__);
2544  DDS *dds = bdds->get_dds();
2545  string container_name = bdds->get_explicit_containers() ? dhi.container->get_symbolic_name(): "";
2546  string filename = dhi.container->access();
2547  DAS* das = 0;
2548  bool das_from_mcache = false;
2549  if(das_cache) {
2550  das = static_cast<DAS*>(das_cache->get(filename));
2551  if(das) {
2552  BESDEBUG(HDF5_NAME, prolog << "DAS Cached hit for : " << filename << endl);
2553  dds->transfer_attributes(das); // no need to copy the cached DAS
2554  das_from_mcache = true;
2555  }
2556  }
2557 
2558  if(false == das_from_mcache) {
2559  das = new DAS;
2560  // This looks at the 'use explicit containers' prop, and if true
2561  // sets the current container for the DAS.
2562  if (!container_name.empty()) das->container_name(container_name);
2563 
2564  hid_t h5_fd =-1;
2565  if (true == _usecf) {
2566  // go to the CF option
2567  h5_fd = H5Fopen(filename.c_str(), H5F_ACC_RDONLY, H5P_DEFAULT);
2568 
2569  read_cfdas( *das,filename,h5_fd);
2570 
2571  H5Fclose(h5_fd);
2572  }
2573  else {
2574  h5_fd = get_fileid(filename.c_str());
2575  find_gloattr(h5_fd, *das);
2576  depth_first(h5_fd, "/", *das);
2577  close_fileid(h5_fd);
2578  }
2579 
2580 
2581  Ancillary::read_ancillary_das(*das, filename);
2582 
2583  dds->transfer_attributes(das);
2584 
2585  // Only free the DAS if it's not added to the cache
2586  if (das_cache) {
2587  // add a copy
2588  BESDEBUG(HDF5_NAME, prolog << "DAS added to the cache for : " << filename << endl);
2589  das_cache->add(das, filename);
2590  }
2591  else {
2592  delete das;
2593  }
2594  }
2595  BESDEBUG(HDF5_NAME, prolog << "Data ACCESS in add_attributes(): set the including attribute flag to true: "<<filename << endl);
2596  bdds->set_ia_flag(true);
2597  return;
2598 
2599 }
2600 
2601 
include the entry functions to execute the handlers
std::string get_symbolic_name() const
retrieve the symbolic name for this container
Definition: BESContainer.h:221
virtual std::string access()=0
returns the true name of this container
Represents an OPeNDAP DAS DAP2 data object within the BES.
virtual void clear_container()
clear the container in the DAP response object
virtual void set_container(const std::string &cn)
set the container in the DAP response object
Holds a DDS object within the BES.
virtual void set_container(const std::string &cn)
set the container in the DAP response object
void set_dds(libdap::DDS *ddsIn)
virtual void clear_container()
clear the container in the DAP response object
libdap::DDS * get_dds()
Represents an OPeNDAP DMR DAP4 data object within the BES.
error object created from libdap error objects and can handle those errors
Definition: BESDapError.h:59
virtual void set_dap4_function(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_dap4_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
virtual void set_constraint(BESDataHandlerInterface &dhi)
set the constraint depending on the context
bool get_explicit_containers() const
Should containers be explicitly represented in the DD* responses?
std::string get_request_xml_base() const
Return the xml:base URL for this request.
Represents an OPeNDAP DataDDS DAP2 data object within the BES.
void set_dds(libdap::DDS *ddsIn)
virtual void set_container(const std::string &cn)
set the container in the DAP response object
virtual void clear_container()
clear the container in the DAP response object
Structure storing information used by the BES to handle the request.
BESContainer * container
pointer to current container in this interface
Abstract exception class for the BES with basic string message.
Definition: BESError.h:58
virtual std::string get_message()
get the error message for this exception
Definition: BESError.h:99
informational response object
Definition: BESInfo.h:63
virtual void add_data(const std::string &s)
add data to this informational object. If buffering is not set then the information is output directl...
Definition: BESInfo.cc:160
exception thrown if internal error encountered
exception thrown if an internal error is found and is fatal to the BES
Represents a specific data type request handler.
virtual BESResponseObject * get_response_object()
return the current response object
Abstract base class representing a specific set of information in response to a request to the BES.
virtual void services_handled(const std::string &handler, std::list< std::string > &services)
returns the list of servies provided by the handler in question
virtual bool start(std::string name)
Definition: BESStopWatch.cc:67
static std::string lowercase(const std::string &s)
Definition: BESUtil.cc:206
static std::string implode(const std::list< std::string > &values, char delim)
Definition: BESUtil.cc:657
An in-memory cache for DapObj (DAS, DDS, ...) objects.
Definition: ObjMemCache.h:84
virtual void add(libdap::DapObj *obj, const std::string &key)
Add an object to the cache and associate it with a key.
Definition: ObjMemCache.cc:63
virtual libdap::DapObj * get(const std::string &key)
Get the cached pointer.
Definition: ObjMemCache.cc:105
void get_value(const std::string &s, std::string &val, bool &found)
Retrieve the value of a given key, if set.
Definition: TheBESKeys.cc:340
static TheBESKeys * TheKeys()
Definition: TheBESKeys.cc:71
Helper functions for generating DAS attributes and a function to check BES Key.
void depth_first(hid_t pid, const char *gname, DAS &das)
Definition: h5das.cc:62
void find_gloattr(hid_t file, DAS &das)
Definition: h5das.cc:487
bool breadth_first(const hid_t file_id, hid_t pid, char *gname, D4Group *par_grp, const char *fname, bool use_dimscale, vector< link_info_t > &hdf5_hls)
Definition: h5dmr.cc:321
hid_t get_fileid(const char *filename)
Definition: h5get.cc:411
void close_fileid(hid_t fid)
Definition: h5get.cc:433
The main header of the HDF5 OPeNDAP handler.