31#include "config_hdf5.h"
37#include <libdap/InternalErr.h>
39#include <libdap/Str.h>
43#include "ObjMemCache.h"
49BaseType *HDF5CFArray::ptr_duplicate()
55bool HDF5CFArray::read()
58 BESDEBUG(
"h5",
"Coming to HDF5CFArray read "<<endl);
62 if((NULL == HDF5RequestHandler::get_lrdata_mem_cache()) &&
63 NULL == HDF5RequestHandler::get_srdata_mem_cache()){
69 short use_cache_flag = 0;
72 if(HDF5RequestHandler::get_srdata_mem_cache() != NULL) {
73 if(((cvtype == CV_EXIST) && (islatlon !=
true)) || (cvtype == CV_NONLATLON_MISS)
74 || (cvtype == CV_FILLINDEX) ||(cvtype == CV_MODIFY) ||(cvtype == CV_SPECIAL)){
76 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
82 if(use_cache_flag !=1) {
84 if(HDF5RequestHandler::get_lrdata_mem_cache() != NULL) {
89 if(HDF5RequestHandler::get_common_cache_dirs() ==
false) {
90 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
91 || (cvtype == CV_EXIST && islatlon ==
true)) {
96 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
104 if(cvtype == CV_LAT_MISS || cvtype == CV_LON_MISS
105 || (cvtype == CV_EXIST && islatlon ==
true)) {
107 vector<string> cur_lrd_non_cache_dir_list;
108 HDF5RequestHandler::get_lrd_non_cache_dir_list(cur_lrd_non_cache_dir_list);
111 if( (cur_lrd_non_cache_dir_list.size() == 0) ||
112 (
"" == check_str_sect_in_list(cur_lrd_non_cache_dir_list,filename,
'/'))) {
115 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
121 vector<string> cur_lrd_var_cache_file_list;
122 HDF5RequestHandler::get_lrd_var_cache_file_list(cur_lrd_var_cache_file_list);
123 if(cur_lrd_var_cache_file_list.size() >0){
128 if(
true == check_var_cache_files(cur_lrd_var_cache_file_list,filename,varname)){
135 if(HDF5CFUtil::cf_dap2_support_numeric_type(dtype,is_dap4)==
true)
143 if(0 == use_cache_flag)
150 if( 3 == use_cache_flag){
151 vector<string> cur_cache_dlist;
152 HDF5RequestHandler::get_lrd_cache_dir_list(cur_cache_dlist);
153 string cache_dir = check_str_sect_in_list(cur_cache_dlist,filename,
'/');
155 cache_key = cache_dir + varname;
157 cache_key = filename + varname;
165 cache_key = filename + varname;
167 handle_data_with_mem_cache(dtype,total_elems,use_cache_flag,cache_key,is_dap4);
180 vector<hsize_t> hoffset;
181 vector<hsize_t>hcount;
182 vector<hsize_t>hstep;
186 throw InternalErr (__FILE__, __LINE__,
187 "The number of dimension of the variable is <=0 for an array.");
193 hoffset.resize(rank);
196 nelms = format_constraint (&offset[0], &step[0], &count[0]);
197 for (
int i = 0; i <rank; i++) {
198 hoffset[i] = (hsize_t) offset[i];
199 hcount[i] = (hsize_t) count[i];
200 hstep[i] = (hsize_t) step[i];
210 bool data_from_disk_cache =
false;
211 bool data_to_disk_cache =
false;
214 bool use_disk_cache = valid_disk_cache();
218 if(
true == use_disk_cache) {
220 BESDEBUG(
"h5",
"Coming to use disk cache "<<endl);
222 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
223 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
224 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
226 string cache_fname=HDF5CFUtil::obtain_cache_fname(diskcache_prefix,filename,varname);
227 cache_fpath = diskcache_dir +
"/"+ cache_fname;
229 int temp_total_elems = 1;
230 for (
unsigned int i = 0; i <dimsizes.size();i++)
231 temp_total_elems = temp_total_elems*dimsizes[i];
232 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
234 if(is_dap4 ==
false && dtype==H5CHAR)
237 int expected_file_size = dtype_size *temp_total_elems;
240 if(
true == disk_cache->get_data_from_cache(cache_fpath, expected_file_size,fd)) {
242 vector<size_t> offset_size_t;
243 offset_size_t.resize(rank);
244 for(
int i = 0; i <rank;i++)
245 offset_size_t[i] = (
size_t)offset[i];
246 size_t offset_1st = INDEX_nD_TO_1D(dimsizes,offset_size_t);
249 for (
int i = 0; i < rank; i++)
250 end[i] = offset[i] +(count[i]-1)*step[i];
251 size_t offset_last = INDEX_nD_TO_1D(dimsizes,end);
256 size_t total_read = dtype_size*(offset_last-offset_1st+1);
258 off_t fpos = lseek(fd,dtype_size*offset_1st,SEEK_SET);
266 data_from_disk_cache = obtain_cached_data(disk_cache,cache_fpath,fd, step,count,total_read,dtype_size);
270 if(
true == data_from_disk_cache)
273 data_to_disk_cache =
true;
279 bool pass_fileid = HDF5RequestHandler::get_pass_fileid();
280 if(
false == pass_fileid) {
281 if ((fileid = H5Fopen(filename.c_str(),H5F_ACC_RDONLY,H5P_DEFAULT))<0) {
283 eherr <<
"HDF5 File " << filename
284 <<
" cannot be opened. "<<endl;
285 throw InternalErr (__FILE__, __LINE__, eherr.str ());
289 if ((dsetid = H5Dopen(fileid,varname.c_str(),H5P_DEFAULT))<0) {
290 HDF5CFUtil::close_fileid(fileid,pass_fileid);
292 eherr <<
"HDF5 dataset " << varname
293 <<
" cannot be opened. "<<endl;
294 throw InternalErr (__FILE__, __LINE__, eherr.str ());
297 if ((dspace = H5Dget_space(dsetid))<0) {
300 HDF5CFUtil::close_fileid(fileid,pass_fileid);
302 eherr <<
"Space id of the HDF5 dataset " << varname
303 <<
" cannot be obtained. "<<endl;
304 throw InternalErr (__FILE__, __LINE__, eherr.str ());
307 if (H5Sselect_hyperslab(dspace, H5S_SELECT_SET,
308 &hoffset[0], &hstep[0],
309 &hcount[0], NULL) < 0) {
313 HDF5CFUtil::close_fileid(fileid,pass_fileid);
315 eherr <<
"The selection of hyperslab of the HDF5 dataset " << varname
317 throw InternalErr (__FILE__, __LINE__, eherr.str ());
320 mspace = H5Screate_simple(rank, &hcount[0],NULL);
324 HDF5CFUtil::close_fileid(fileid,pass_fileid);
326 eherr <<
"The creation of the memory space of the HDF5 dataset " << varname
328 throw InternalErr (__FILE__, __LINE__, eherr.str ());
332 if ((dtypeid = H5Dget_type(dsetid)) < 0) {
337 HDF5CFUtil::close_fileid(fileid,pass_fileid);
339 eherr <<
"Obtaining the datatype of the HDF5 dataset " << varname
341 throw InternalErr (__FILE__, __LINE__, eherr.str ());
345 if ((memtype = H5Tget_native_type(dtypeid, H5T_DIR_ASCEND))<0) {
351 HDF5CFUtil::close_fileid(fileid,pass_fileid);
353 eherr <<
"Obtaining the memory type of the HDF5 dataset " << varname
355 throw InternalErr (__FILE__, __LINE__, eherr.str ());
363 if(
true == add_mem_cache) {
369 HDF5CFUtil::close_fileid(fileid,pass_fileid);
370 throw InternalErr(__FILE__,__LINE__,
"The memory data cache buffer needs to be set");
372 read_ret= H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,buf);
378 HDF5CFUtil::close_fileid(fileid,pass_fileid);
379 throw InternalErr(__FILE__,__LINE__,
"Cannot read the data to the buffer.");
395 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
403 HDF5CFUtil::close_fileid(fileid,pass_fileid);
405 eherr <<
"Cannot read the HDF5 dataset " << varname
406 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
407 throw InternalErr (__FILE__, __LINE__, eherr.str ());
412 set_value((dods_int8 *)&val[0],nelms);
416 newval.resize(nelms);
418 for (
int counter = 0; counter < nelms; counter++)
419 newval[counter] = (
short) (val[counter]);
421 set_value ((dods_int16 *) &newval[0], nelms);
424 if(
true == data_to_disk_cache) {
426 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
427 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,2,val,nelms);
435 HDF5CFUtil::close_fileid(fileid,pass_fileid);
437 eherr <<
"write data to cache failed.";
438 throw InternalErr (__FILE__, __LINE__, eherr.str ());
459 size_t dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
461 val.resize(nelms*dtype_size);
463 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
470 HDF5CFUtil::close_fileid(fileid,pass_fileid);
472 eherr <<
"Cannot read the HDF5 dataset " << varname
473 <<
" with the type of H5T_NATIVE_UCHAR "<<endl;
474 throw InternalErr (__FILE__, __LINE__, eherr.str ());
481 if(
true == data_to_disk_cache) {
482 BESDEBUG(
"h5",
"writing data to disk cache "<<endl);
484 write_data_to_cache(dsetid,dspace,mspace,memtype,cache_fpath,dtype_size,val,nelms);
492 HDF5CFUtil::close_fileid(fileid,pass_fileid);
494 eherr <<
"write data to cache failed.";
495 throw InternalErr (__FILE__, __LINE__, eherr.str ());
511 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
519 HDF5CFUtil::close_fileid(fileid,pass_fileid);
522 eherr <<
"Cannot read the HDF5 dataset " << varname
523 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
524 throw InternalErr (__FILE__, __LINE__, eherr.str ());
527 set_value ((dods_int16 *) &val[0], nelms);
534 vector<unsigned short> val;
536 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
544 HDF5CFUtil::close_fileid(fileid,pass_fileid);
546 eherr <<
"Cannot read the HDF5 dataset " << varname
547 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
548 throw InternalErr (__FILE__, __LINE__, eherr.str ());
551 set_value ((dods_uint16 *) &val[0], nelms);
560 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
567 HDF5CFUtil::close_fileid(fileid,pass_fileid);
569 eherr <<
"Cannot read the HDF5 dataset " << varname
570 <<
" with the type of H5T_NATIVE_INT "<<endl;
571 throw InternalErr (__FILE__, __LINE__, eherr.str ());
574 set_value ((dods_int32 *) &val[0], nelms);
580 vector<unsigned int>val;
582 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
589 HDF5CFUtil::close_fileid(fileid,pass_fileid);
591 eherr <<
"Cannot read the HDF5 dataset " << varname
592 <<
" with the type of H5T_NATIVE_UINT "<<endl;
593 throw InternalErr (__FILE__, __LINE__, eherr.str ());
596 set_value ((dods_uint32 *) &val[0], nelms);
606 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
613 HDF5CFUtil::close_fileid(fileid,pass_fileid);
615 eherr <<
"Cannot read the HDF5 dataset " << varname
616 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
617 throw InternalErr (__FILE__, __LINE__, eherr.str ());
620 set_value ((dods_float32 *) &val[0], nelms);
630 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
638 HDF5CFUtil::close_fileid(fileid,pass_fileid);
640 eherr <<
"Cannot read the HDF5 dataset " << varname
641 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
642 throw InternalErr (__FILE__, __LINE__, eherr.str ());
645 set_value ((dods_float64 *) &val[0], nelms);
653 size_t ty_size = H5Tget_size(dtypeid);
660 HDF5CFUtil::close_fileid(fileid,pass_fileid);
662 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
664 throw InternalErr (__FILE__, __LINE__, eherr.str ());
667 vector <char> strval;
668 strval.resize(nelms*ty_size);
669 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)&strval[0]);
677 HDF5CFUtil::close_fileid(fileid,pass_fileid);
679 eherr <<
"Cannot read the HDF5 dataset " << varname
680 <<
" with the type of the fixed size HDF5 string "<<endl;
681 throw InternalErr (__FILE__, __LINE__, eherr.str ());
684 string total_string(strval.begin(),strval.end());
686 vector <string> finstrval;
687 finstrval.resize(nelms);
688 for (
int i = 0; i<nelms; i++)
689 finstrval[i] = total_string.substr(i*ty_size,ty_size);
696 if ((
true == HDF5RequestHandler::get_drop_long_string()) &&
697 ty_size > NC_JAVA_STR_SIZE_LIMIT) {
698 for (
int i = 0; i<nelms; i++)
701 set_value(finstrval,nelms);
702 total_string.clear();
709 size_t ty_size = H5Tget_size(memtype);
716 HDF5CFUtil::close_fileid(fileid,pass_fileid);
718 eherr <<
"Cannot obtain the size of the fixed size HDF5 string of the dataset "
720 throw InternalErr (__FILE__, __LINE__, eherr.str ());
722 vector <char> strval;
723 strval.resize(nelms*ty_size);
724 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,(
void*)&strval[0]);
732 HDF5CFUtil::close_fileid(fileid,pass_fileid);
734 eherr <<
"Cannot read the HDF5 dataset " << varname
735 <<
" with the type of the HDF5 variable length string "<<endl;
736 throw InternalErr (__FILE__, __LINE__, eherr.str ());
739 vector<string>finstrval;
740 finstrval.resize(nelms);
741 char*temp_bp = &strval[0];
742 char*onestring = NULL;
743 for (
int i =0;i<nelms;i++) {
744 onestring = *(
char**)temp_bp;
746 finstrval[i] =string(onestring);
753 if (
false == strval.empty()) {
754 herr_t ret_vlen_claim;
755 ret_vlen_claim = H5Dvlen_reclaim(memtype,mspace,H5P_DEFAULT,(
void*)&strval[0]);
756 if (ret_vlen_claim < 0){
762 HDF5CFUtil::close_fileid(fileid,pass_fileid);
764 eherr <<
"Cannot reclaim the memory buffer of the HDF5 variable length string of the dataset "
766 throw InternalErr (__FILE__, __LINE__, eherr.str ());
774 if (
true == HDF5RequestHandler::get_drop_long_string()) {
775 bool drop_long_str =
false;
776 for (
int i =0;i<nelms;i++) {
777 if(finstrval[i].size() >NC_JAVA_STR_SIZE_LIMIT){
778 drop_long_str =
true;
782 if (drop_long_str ==
true) {
783 for (
int i =0;i<nelms;i++)
787 set_value(finstrval,nelms);
799 HDF5CFUtil::close_fileid(fileid,pass_fileid);
801 eherr <<
"Cannot read the HDF5 dataset " << varname
802 <<
" with the unsupported HDF5 datatype"<<endl;
803 throw InternalErr (__FILE__, __LINE__, eherr.str ());
812 HDF5CFUtil::close_fileid(fileid,pass_fileid);
817bool HDF5CFArray::valid_disk_cache() {
819 bool ret_value =
false;
820 if(
true == HDF5RequestHandler::get_use_disk_cache()) {
822 BESDEBUG(
"h5",
"Coming to disk cache "<<endl);
824 if(dtype == H5CHAR || dtype ==H5UCHAR || dtype==H5INT16 || dtype ==H5UINT16 ||
825 dtype == H5INT32 || dtype ==H5UINT32 || dtype ==H5FLOAT32 || dtype==H5FLOAT64 ||
826 dtype == H5INT64 || dtype ==H5UINT64){
828 BESDEBUG(
"h5",
"Coming to disk cache datatype block"<<endl);
830 string diskcache_dir = HDF5RequestHandler::get_disk_cache_dir();
831 string diskcache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
832 long diskcache_size = HDF5RequestHandler::get_disk_cache_size();
834 if((
"" == diskcache_dir)||(
""==diskcache_prefix)||(diskcache_size <=0))
835 throw InternalErr (__FILE__, __LINE__,
"Either the cached dir is empty or the prefix is NULL or the cache size is not set.");
838 if(stat(diskcache_dir.c_str(),&sb) !=0) {
839 string err_mesg=
"The cached directory " + diskcache_dir;
840 err_mesg = err_mesg +
" doesn't exist. ";
841 throw InternalErr(__FILE__,__LINE__,err_mesg);
844 if(
true == S_ISDIR(sb.st_mode)) {
845 if(access(diskcache_dir.c_str(),R_OK|W_OK|X_OK) == -1) {
846 string err_mesg=
"The cached directory " + diskcache_dir;
847 err_mesg = err_mesg +
" can NOT be read,written or executable.";
848 throw InternalErr(__FILE__,__LINE__,err_mesg);
852 string err_mesg=
"The cached directory " + diskcache_dir;
853 err_mesg = err_mesg +
" is not a directory.";
854 throw InternalErr(__FILE__,__LINE__,err_mesg);
859 short dtype_size = HDF5CFUtil::H5_numeric_atomic_type_size(dtype);
861 if(
true == HDF5RequestHandler::get_disk_cache_comp_data()){
862 BESDEBUG(
"h5",
"Compression disk cache key is true"<<endl);
863 ret_value = valid_disk_cache_for_compressed_data(dtype_size);
864 BESDEBUG(
"h5",
"variable disk cache passes the compression parameter check"<<endl);
867 BESDEBUG(
"h5",
"Compression disk cache key is NOT set, disk cache key is true."<<endl);
877bool HDF5CFArray:: valid_disk_cache_for_compressed_data(
short dtype_size)
const {
879 bool ret_value =
false;
883 size_t total_byte = total_elems*dtype_size;
885 if((comp_ratio < HDF5RequestHandler::get_disk_comp_threshold())
886 && (total_elems*dtype_size >= HDF5RequestHandler::get_disk_var_size())) {
887 if(
true == HDF5RequestHandler::get_disk_cache_float_only_comp()) {
888 if(dtype==H5FLOAT32 || dtype == H5FLOAT64)
898bool HDF5CFArray::obtain_cached_data(
HDF5DiskCache *disk_cache,
const string & cache_fpath,
int fd,vector<int> &cd_step, vector<int>&cd_count,
size_t total_read,
short dtype_size) {
900 ssize_t ret_read_val = -1;
903 buf.resize(total_read);
906 if((-1 == ret_read_val) || (ret_read_val != (ssize_t)total_read)) {
911 unsigned int nele_to_read = 1;
912 for(
int i = 0; i<rank;i++)
913 nele_to_read *=cd_count[i];
915 if(nele_to_read == (total_read/dtype_size)) {
921 vector<int>cd_start(rank,0);
922 vector<size_t>cd_pos(rank,0);
923 int nelms_to_send = 1;
924 for(
int i = 0; i <rank; i++)
925 nelms_to_send = nelms_to_send*cd_count[i];
932 vector<int>total_val;
933 total_val.resize(total_read/dtype_size);
934 memcpy(&total_val[0],(
void*)&buf[0],total_read);
936 vector<int>final_val;
951 if(is_dap4 ==
false) {
952 vector<short>final_val;
964 set_value((dods_int16*)&final_val[0],nelms_to_send);
967 vector<char>final_val;
979 set_value((dods_int8*)&final_val[0],nelms_to_send);
988 vector<unsigned char>total_val;
989 total_val.resize(total_read/dtype_size);
990 memcpy(&total_val[0],(
void*)&buf[0],total_read);
992 vector<unsigned char>final_val;
993 subset<unsigned char>(
1006 vector<unsigned char>final_val;
1007 subset<unsigned char>(
1019 set_value ((dods_byte *) &final_val[0], nelms_to_send);
1026 vector<short>total_val;
1027 total_val.resize(total_read/dtype_size);
1028 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1030 vector<short>final_val;
1044 vector<short>final_val;
1057 set_value ((dods_int16 *) &final_val[0], nelms_to_send);
1064 vector<unsigned short>total_val;
1065 total_val.resize(total_read/dtype_size);
1066 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1068 vector<unsigned short>final_val;
1069 subset<unsigned short>(
1082 vector<unsigned short>final_val;
1083 subset<unsigned short>(
1095 set_value ((dods_uint16 *) &final_val[0], nelms_to_send);
1102 vector<int>total_val;
1103 total_val.resize(total_read/dtype_size);
1104 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1106 vector<int>final_val;
1121 vector<int>final_val;
1135 set_value ((dods_int32 *) &final_val[0], nelms_to_send);
1142 vector<unsigned int>total_val;
1143 total_val.resize(total_read/dtype_size);
1144 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1146 vector<unsigned int>final_val;
1147 subset<unsigned int>(
1160 vector<unsigned int>final_val;
1161 subset<unsigned int>(
1173 set_value ((dods_uint32 *) &final_val[0], nelms_to_send);
1180 vector<unsigned int>total_val;
1181 total_val.resize(total_read/dtype_size);
1182 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1184 vector<unsigned int>final_val;
1185 subset<unsigned int>(
1198 vector<long long >final_val;
1211 set_value ((dods_int64 *) &final_val[0], nelms_to_send);
1220 vector<unsigned int>total_val;
1221 total_val.resize(total_read/dtype_size);
1222 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1224 vector<unsigned int>final_val;
1225 subset<unsigned int>(
1238 vector<unsigned long long >final_val;
1239 subset<unsigned long long >(
1251 set_value ((dods_uint64 *) &final_val[0], nelms_to_send);
1259 vector<float>total_val;
1260 total_val.resize(total_read/dtype_size);
1261 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1263 vector<float>final_val;
1277 vector<float>final_val;
1291 set_value ((dods_float32 *) &final_val[0], nelms_to_send);
1297 vector<double>total_val;
1298 total_val.resize(total_read/dtype_size);
1299 memcpy(&total_val[0],(
void*)&buf[0],total_read);
1301 vector<double>final_val;
1314 vector<double>final_val;
1327 set_value ((dods_float64 *) &final_val[0], nelms_to_send);
1331 throw InternalErr (__FILE__, __LINE__,
"unsupported data type.");
1341HDF5CFArray::write_data_to_cache(hid_t dset_id, hid_t , hid_t , hid_t memtype,
1342 const string& cache_fpath,
short dtype_size,
const vector<char> &buf,
int nelms) {
1344 unsigned long long disk_cache_size = HDF5RequestHandler::get_disk_cache_size();
1345 string disk_cache_dir = HDF5RequestHandler::get_disk_cache_dir();
1346 string disk_cache_prefix = HDF5RequestHandler::get_disk_cachefile_prefix();
1348 int total_nelem = 1;
1349 for(
int i = 0; i <rank; i++)
1350 total_nelem = total_nelem*dimsizes[i];
1354 if(H5CHAR == dtype && is_dap4 ==
false) {
1356 vector<short>newval;
1357 newval.resize(total_nelem);
1358 if(total_nelem == nelms) {
1359 for (
int i = 0; i < total_nelem;i++)
1360 newval[i] = (
short)buf[i];
1361 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)&newval[0]);
1365 val2.resize(total_nelem);
1366 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val2[0])<0)
1367 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole HDF5 dataset for the disk cache.");
1368 for (
int i = 0; i < total_nelem;i++)
1369 newval[i] = (
short)val2[i];
1370 disk_cache->write_cached_data2(cache_fpath,
sizeof(
short)*total_nelem,(
const void*)&newval[0]);
1374 if(total_nelem == nelms) {
1375 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&buf[0]);
1378 val.resize(dtype_size*total_nelem);
1379 if(H5Dread(dset_id, memtype, H5S_ALL, H5S_ALL,H5P_DEFAULT, &val[0])<0)
1380 throw InternalErr (__FILE__, __LINE__,
"Cannot read the whole SDS for cache.");
1382 disk_cache->write_cached_data2(cache_fpath,dtype_size*total_nelem,(
const void*)&val[0]);
1388void HDF5CFArray::read_data_from_mem_cache(
void*buf) {
1393 int nelms = format_constraint (&offset[0], &step[0], &count[0]);
1395 vector<int>at_pos(at_ndims,0);
1396 for (
int i = 0; i< rank; i++)
1397 at_pos[i] = at_offset[i];
1405 vector<unsigned char> val;
1407 subset<unsigned char>(
1420 set_value ((dods_byte *) &val[0], nelms);
1432 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1434 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1444 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1445 ostringstream eherr;
1446 eherr <<
"Cannot read the HDF5 dataset " << varname
1447 <<
" with the type of H5T_NATIVE_CHAR "<<endl;
1448 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1452 vector<short>newval;
1453 newval.resize(nelms);
1455 for (
int counter = 0; counter < nelms; counter++)
1456 newval[counter] = (
short) (val[counter]);
1458 set_value ((dods_int16 *) &newval[0], nelms);
1469 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1471 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1481 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1483 ostringstream eherr;
1484 eherr <<
"Cannot read the HDF5 dataset " << varname
1485 <<
" with the type of H5T_NATIVE_SHORT "<<endl;
1486 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1489 set_value ((dods_int16 *) &val[0], nelms);
1496 vector<unsigned short> val;
1499 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1501 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1505 if (rank > 0) H5Sclose(mspace);
1510 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1511 ostringstream eherr;
1512 eherr <<
"Cannot read the HDF5 dataset " << varname
1513 <<
" with the type of H5T_NATIVE_USHORT "<<endl;
1514 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1517 set_value ((dods_uint16 *) &val[0], nelms);
1527 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1529 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1538 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1539 ostringstream eherr;
1540 eherr <<
"Cannot read the HDF5 dataset " << varname
1541 <<
" with the type of H5T_NATIVE_INT "<<endl;
1542 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1545 set_value ((dods_int32 *) &val[0], nelms);
1551 vector<unsigned int>val;
1554 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1556 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1566 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1567 ostringstream eherr;
1568 eherr <<
"Cannot read the HDF5 dataset " << varname
1569 <<
" with the type of H5T_NATIVE_UINT "<<endl;
1570 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1573 set_value ((dods_uint32 *) &val[0], nelms);
1584 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1586 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1595 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1596 ostringstream eherr;
1597 eherr <<
"Cannot read the HDF5 dataset " << varname
1598 <<
" with the type of H5T_NATIVE_FLOAT "<<endl;
1599 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1602 set_value ((dods_float32 *) &val[0], nelms);
1613 read_ret = H5Dread(dsetid,memtype,H5S_ALL,H5S_ALL,H5P_DEFAULT,&val[0]);
1615 read_ret = H5Dread(dsetid,memtype,mspace,dspace,H5P_DEFAULT,&val[0]);
1624 HDF5CFUtil::close_fileid(fileid,pass_fileid);
1625 ostringstream eherr;
1626 eherr <<
"Cannot read the HDF5 dataset " << varname
1627 <<
" with the type of H5T_NATIVE_DOUBLE "<<endl;
1628 throw InternalErr (__FILE__, __LINE__, eherr.str ());
1631 set_value ((dods_float64 *) &val[0], nelms);
1646BaseType* HDF5CFArray::h5cfdims_transform_to_dap4(D4Group *grp) {
1650 Array *dest =
static_cast<HDF5CFArray*
>(ptr_duplicate());
1656 D4Dimensions *grp_dims = grp->dims();
1657 for (Array::Dim_iter dap2_dim = dest->dim_begin(), e = dest->dim_end(); dap2_dim != e; ++dap2_dim) {
1658 if (!(*dap2_dim).name.empty()) {
1661 D4Dimension *d4_dim = grp_dims->find_dim((*dap2_dim).name);
1663 d4_dim =
new D4Dimension((*dap2_dim).name, (*dap2_dim).size);
1664 grp_dims->add_dim_nocopy(d4_dim);
1668 (*dap2_dim).dim = d4_dim;
1680BaseType* HDF5CFArray::h5cfdims_transform_to_dap4_int64(D4Group *grp) {
1684 Array *dest =
static_cast<HDF5CFArray*
>(ptr_duplicate());
1690 for (Array::Dim_iter d = dest->dim_begin(), e = dest->dim_end(); d != e; ++d) {
1691 if (
false == (*d).name.empty()) {
1693 D4Group *temp_grp = grp;
1694 D4Dimension *d4_dim = NULL;
1697 D4Dimensions *temp_dims = temp_grp->dims();
1700 d4_dim = temp_dims->find_dim((*d).name);
1706 if(temp_grp->get_parent())
1707 temp_grp =
static_cast<D4Group*
>(temp_grp->get_parent());
1716 bool d4_dim_null = ((d4_dim==NULL)?
true:
false);
1721 if(d4_dim_null ==
true) {
1723 d4_dim =
new D4Dimension((*d).name, (*d).size);
1724 D4Dimensions * dims = grp->dims();
1725 dims->add_dim_nocopy(d4_dim);
1731 dest->set_is_dap4(
true);
1740HDF5CFArray::format_constraint (
int *offset,
int *step,
int *count)
1746 Dim_iter p = dim_begin ();
1748 while (p != dim_end ()) {
1750 int start = dimension_start (p,
true);
1751 int stride = dimension_stride (p,
true);
1752 int stop = dimension_stop (p,
true);
1758 oss <<
"Array/Grid hyperslab start point "<< start <<
1759 " is greater than stop point " << stop <<
".";
1760 throw Error(malformed_expr, oss.str());
1765 count[id] = ((stop - start) / stride) + 1;
1769 "=format_constraint():"
1770 <<
"id=" <<
id <<
" offset=" << offset[
id]
1771 <<
" step=" << step[
id]
1772 <<
" count=" << count[
id]
This class includes the methods to read data array into DAP buffer from an HDF5 dataset for the CF op...
include the entry functions to execute the handlers
virtual void unlock_and_close(const std::string &target)
virtual void purge_file(const std::string &file)
Purge a single file from the cache.
virtual void read_data_NOT_from_mem_cache(bool add_cache, void *buf)
static HDF5DiskCache * get_instance(const long, const std::string &, const std::string &)
Helper functions for generating DAS attributes and a function to check BES Key.
static ssize_t read_buffer_from_file(int fd, void *buf, size_t)
Getting a subset of a variable.