| | 1 | = Data Publishing = |
| | 2 | |
| | 3 | == Configuring a new project for ESGF publication == |
| | 4 | |
| | 5 | |
| | 6 | [[NoteBox(tip, See the [http://esg-pcmdi.llnl.gov/internal/esg-data-node-documentation/introduction-to-esg-data-node-configuration/ ESGF] publication reference for details.)]] |
| | 7 | |
| | 8 | |
| | 9 | In order to publish, you have to configure a text file, `/esg/config/esgcet/esg.ini`. For this propose, we are going to create a new project called `cordex`: |
| | 10 | |
| | 11 | |
| | 12 | {{{ |
| | 13 | #!sh |
| | 14 | [initialize] |
| | 15 | log_level = DEBUG |
| | 16 | initial_models_table = /esg/config/esgcet/esgcet_models_table.txt |
| | 17 | |
| | 18 | |
| | 19 | [DEFAULT] |
| | 20 | thredds_dataset_roots = |
| | 21 | esg_dataroot | /datasets |
| | 22 | |
| | 23 | project_options = |
| | 24 | cordex | CORDEX Output data | 1 |
| | 25 | [project:cordex] |
| | 26 | # name | category_type | is_mandatory | is_thredds_property | display_order |
| | 27 | categories = |
| | 28 | project | enum | true | true | 0 |
| | 29 | domain | enum | true | true | 1 |
| | 30 | institute | enum | true | true | 2 |
| | 31 | driving_model | enum | false | true | 3 |
| | 32 | experiment | enum | false | true | 4 |
| | 33 | ensemble | enum | false | true | 5 |
| | 34 | model | enum | false | true | 6 |
| | 35 | time_frequency | enum | false | true | 7 |
| | 36 | version | enum | false | true | 8 |
| | 37 | rcm_model | enum | false | true | 9 |
| | 38 | rcm_version | enum | false | true | 10 |
| | 39 | description | text | false | false | 99 |
| | 40 | category_defaults = |
| | 41 | domain | EUR-22 |
| | 42 | institute | UCAN |
| | 43 | driving_model | ERAINT |
| | 44 | ensemble | r1i1p1 |
| | 45 | model | WRF331G |
| | 46 | time_frequency| mon |
| | 47 | dataset_id = cordex.%(domain)s.%(institute)s.%(driving_model)s.%(experiment)s.%(ensemble)s.WRF331G_v02.%(time_frequency)s.%(variable)s |
| | 48 | directory_format = /datasets/CORDEX/output/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(rcm_model)s/%(rcm_version)s/%(time_frequency)s/%(variable)s/%(version)s |
| | 49 | domain_map = map(project_id,domain : domain_description) |
| | 50 | cordex | SAM-44 | South America |
| | 51 | cordex | CAM-44 | Central America |
| | 52 | cordex | NAM-44 | North America |
| | 53 | cordex | EUR-44 | Europe |
| | 54 | cordex | EUR-22 | Europe |
| | 55 | cordex | AFR-44 | Africa |
| | 56 | cordex | WAS-44 | West Asia |
| | 57 | cordex | EAS-44 | East Asia |
| | 58 | cordex | CAS-44 | Central Asia |
| | 59 | cordex | AUS-44 | Australasia |
| | 60 | cordex | ANT-44 | Antarctica |
| | 61 | cordex | ARC-44 | The Arctic |
| | 62 | cordex | MED-44 | HYMEX Mediterranean |
| | 63 | cordex | EUR-11 | High-res. Europe |
| | 64 | cordex | SAM-44i | South America |
| | 65 | cordex | CAM-44i | Central America |
| | 66 | cordex | NAM-44i | North America |
| | 67 | cordex | EUR-44i | Europe |
| | 68 | cordex | AFR-44i | Africa |
| | 69 | cordex | WAS-44i | West Asia |
| | 70 | cordex | EAS-44i | East Asia |
| | 71 | cordex | CAS-44i | Central Asia |
| | 72 | cordex | AUS-44i | Australasia |
| | 73 | cordex | ANT-44i | Antarctica |
| | 74 | cordex | ARC-44i | The Arctic |
| | 75 | cordex | MED-44i | HYMEX Mediterranean |
| | 76 | cordex | EUR-11i | High-res. Europe |
| | 77 | cordex | MNA-44 | Middle East and North Africa |
| | 78 | cordex | MNA-44i | Middle East and North Africa |
| | 79 | cordex | MNA-22 | Middle East and North Africa |
| | 80 | cordex | MNA-22i | Middle East and North Africa |
| | 81 | domain_options = SAM-44,CAM-44,NAM-44,EUR-44,EUR-22,EUR-44i,AFR-44,AFR-44i,WAS-44,EAS-44,CAS-44,AUS-44,ANT-44,ARC-44,MED-44,EUR-11,SAM-44i,CAM-44i,NAM-44i,EUR-44i,AFR-44i,WAS-44i,EAS-44i,CAS-44i,AUS-44i,ANT-44i,ARC-44i,MED-44i,EUR-11i,MNA-44,MNA-44i,MNA-22,MNA-22i |
| | 82 | driving_model_options = ERAINT, ECMWF-ERAINT, CCCma-CanESM2, CNRM-CERFACS-CNRM-CM5, ICHEC-EC-EARTH, MIROC-MIROC5, MOHC-HadGEM2-ES, MPI-M-MPI-ESM-LR, NCC-NorESM1-M, NOAA-GFDL-GFDL-ESM2M, IPSL-IPSL-CM5A-MR |
| | 83 | ensemble_options = r1i1p1, r12i1p1, r0i0p0 |
| | 84 | experiment_options = |
| | 85 | cordex | evaluation | no description |
| | 86 | cordex | historical | no description |
| | 87 | cordex | rcp4 | no description |
| | 88 | cordex | rcp26 | no description |
| | 89 | cordex | rcp45 | no description |
| | 90 | cordex | rcp85 | no description |
| | 91 | institute_map = map(project_id,model : institute) |
| | 92 | cordex | WRF331G | UCAN |
| | 93 | institute_options = UCAN |
| | 94 | las_configure = false |
| | 95 | las_time_delta_map = map(time_frequency : las_time_delta) |
| | 96 | mon | 1 month |
| | 97 | day | 1 day |
| | 98 | fx | fixed |
| | 99 | sem | semi |
| | 100 | maps = institute_map, las_time_delta_map, domain_map |
| | 101 | model_options = WRF331G |
| | 102 | parent_id = wdcc2.cordex |
| | 103 | project_handler_name = basic_builtin |
| | 104 | rcm_model_options = UCAN-WRF331G |
| | 105 | rcm_version_options = v02 |
| | 106 | thredds_exclude_variables = a, a_bnds, alev1, alevel, alevhalf, alt40, b, b_bnds, basin, bnds, bounds_lat, bounds_lon, dbze, depth, depth0m, depth100m, depth_bnds, geo_region, height, height10m, height2m, Lambert_Conformal, lat, lat_bnds, lat_bounds, latitude, latitude_bnds, layer, lev, lev_bnds, location, lon, lon_bnds, lon_bounds, longitude, longitude_bnds, olayer100m, olevel, oline, p0, p220, p500, p560, p700, p840, plev, plev3, plev7, plev8, plev_bnds, plevs, pressure1, region, rho, rlat, rotated_pole, rlon, scatratio, sdepth, sdepth1, sza5, tau, tau_bnds, time, time1, time2, time_bnds, vegtype, x, y |
| | 107 | time_frequency_options = day,fx,mon,sem,3hr,6hr |
| | 108 | variable_locate = ps,ps_ | vas,vas_ | uas,uas_ |tasmin,tasmin_|tasmax,tasmax_| tas,tas_ | sfcWindmax,sfcWindmax_ | sfcWind,sfcWind_ |rsds,rsds_|rlds,rlds_| pr,pr_ | prc,prc_| mrso,mrso_| mrros,mrros_|huss,huss_| hfss,hfss_|hfls,hfls_ |
| | 109 | variable_per_file = true |
| | 110 | version_options = 20131108 |
| | 111 | }}} |
| | 112 | |
| | 113 | Therefore, if you use the above configuration file, you will have to create a tree directory like this: |
| | 114 | |
| | 115 | {{{ |
| | 116 | |
| | 117 | [esgf@data datasets]# tree |
| | 118 | . |
| | 119 | `-- CORDEX |
| | 120 | `-- output |
| | 121 | `-- EUR-22 |
| | 122 | `-- UCAN |
| | 123 | `-- ECMWF-ERAINT |
| | 124 | `-- evaluation |
| | 125 | `-- r1i1p1 |
| | 126 | `-- UCAN-WRF311G |
| | 127 | `-- v02 |
| | 128 | |-- 3hr |
| | 129 | | |-- hfls |
| | 130 | | | `-- 20131108 |
| | 131 | | | |-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 132 | | | `-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 133 | | |-- hfss |
| | 134 | | | `-- 20131108 |
| | 135 | | | |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 136 | | | |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc |
| | 137 | | | `-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 138 | | |-- huss |
| | 139 | | | `-- 20131108 |
| | 140 | | | |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 141 | | | |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc |
| | 142 | | | `-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 143 | | |-- pr |
| | 144 | | | `-- 20131108 |
| | 145 | | | |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 146 | | | `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 147 | | |-- prc |
| | 148 | | | `-- 20131108 |
| | 149 | | | |-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 150 | | | `-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 151 | | |-- ps |
| | 152 | | | `-- 20131108 |
| | 153 | | | |-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 154 | | | `-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 155 | | |-- rlds |
| | 156 | | | `-- 20131108 |
| | 157 | | | |-- rlds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 158 | | | |-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc |
| | 159 | | | `-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 160 | | |-- sfcWind |
| | 161 | | | `-- 20131108 |
| | 162 | | | |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 163 | | | |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc |
| | 164 | | | |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc |
| | 165 | | | `-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 166 | | `-- tas |
| | 167 | | `-- 20131108 |
| | 168 | | |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| | 169 | | `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| | 170 | |-- 6hr |
| | 171 | | |-- mrros |
| | 172 | | | `-- 20131108 |
| | 173 | | | |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc |
| | 174 | | | |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc |
| | 175 | | | `-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc |
| | 176 | | |-- uas |
| | 177 | | | `-- 20131108 |
| | 178 | | | |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc |
| | 179 | | | |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc |
| | 180 | | | `-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc |
| | 181 | | `-- vas |
| | 182 | | `-- 20131108 |
| | 183 | | |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc |
| | 184 | | |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc |
| | 185 | | `-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc |
| | 186 | |-- day |
| | 187 | | |-- mrso |
| | 188 | | | `-- 20131108 |
| | 189 | | | |-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| | 190 | | | `-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| | 191 | | |-- pr |
| | 192 | | | `-- 20131108 |
| | 193 | | | |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| | 194 | | | `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| | 195 | | |-- sfcWindmax |
| | 196 | | | `-- 20131108 |
| | 197 | | | |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| | 198 | | | |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20010101-20051231.nc |
| | 199 | | | `-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| | 200 | | |-- tasmax |
| | 201 | | | `-- 20131108 |
| | 202 | | | |-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| | 203 | | | `-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| | 204 | | `-- tasmin |
| | 205 | | `-- 20131108 |
| | 206 | | |-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| | 207 | | `-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| | 208 | `-- mon |
| | 209 | |-- pr |
| | 210 | | `-- 20131108 |
| | 211 | | |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc |
| | 212 | | `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc |
| | 213 | `-- tas |
| | 214 | `-- 20131108 |
| | 215 | |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc |
| | 216 | |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19910101-20001231.nc |
| | 217 | `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc |
| | 218 | |
| | 219 | |
| | 220 | }}} |
| | 221 | |
| | 222 | |
| | 223 | Then you have to add the project name to the `esgcet_models_table.txt` file |
| | 224 | |
| | 225 | {{{ |
| | 226 | #!sh |
| | 227 | $ echo " cordex | WRF331G | http://meteo.unican.es | UNICAN WRF3.3.1 Model version, 2.0" >> /esg/config/esgcet/esgcet_models_table.txt |
| | 228 | }}} |
| | 229 | |
| | 230 | |
| | 231 | After modifying `esgcet_models_table.txt` and `esg.ini` files, you have to update the data base by executing : |
| | 232 | {{{ |
| | 233 | #!sh |
| | 234 | $ cd /usr/local/uvcdat/1.4.0/bin/ |
| | 235 | $ ./esginitialize -i /esg/config/esgcet/esg.ini -c |
| | 236 | }}} |
| | 237 | |
| | 238 | * `./esginitialize -d 0`: To remove all tables |
| | 239 | |
| | 240 | * `./esginitialize -c` : Upgrade the database schema to the latest version, and initialize projects, models, experiments, and standard names from configuration files. |
| | 241 | |
| | 242 | == Using the ESGF Publisher == |
| | 243 | |
| | 244 | |
| | 245 | This takes place in three steps: |
| | 246 | |
| | 247 | * Scan each file for metadata and save the metadata in the node database. (This is in contrast to running `esgscan_directory`, which just scans the directory structure.) |
| | 248 | |
| | 249 | * Generate a THREDDS catalog based on the scanned information. THREDDS is a data and metadata server used by ESGF. |
| | 250 | |
| | 251 | * Notify the idx that one or more catalogs have been generated. |
| | 252 | |
| | 253 | === File Scan Phase === |
| | 254 | |
| | 255 | In order to scan the cordex files for metadata, run `esgscan_directory` to generate a mapfile and after that run `esgpublish` with input from a mapfile: |
| | 256 | {{{ |
| | 257 | #!sh |
| | 258 | $ whoami |
| | 259 | esgf |
| | 260 | $ cd /usr/local/uvcdat/1.4.0/bin |
| | 261 | $ ./esgscan_directory -i /esg/config/esgcet/esg.ini --project cordex -o ~/cordex.txt /datasets/CORDEX |
| | 262 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt |
| | 263 | }}} |
| | 264 | |
| | 265 | In order to check that the datasets were entered correctly into the database, list the datasets for cordex project: |
| | 266 | |
| | 267 | {{{ |
| | 268 | #!sh |
| | 269 | $ ./esglist_datasets -i /esg/config/esgcet/esg.ini cordex |
| | 270 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |
| | 271 | | id | name | project | model | experiment | run_name | offline | master_gateway | domain | institute | publish_time | publish_status | parent | version | version_name | comment | |
| | 272 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |
| | 273 | | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:21 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1 | None | |
| | 274 | | 2 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:25 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1 | None | |
| | 275 | | 3 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:29 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1 | None | |
| | 276 | | 4 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:33 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1 | None | |
| | 277 | | 5 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:37 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1 | None | |
| | 278 | | 6 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:40 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps.v1 | None | |
| | 279 | | 7 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:44 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds.v1 | None | |
| | 280 | | 8 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:48 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds.v1 | None | |
| | 281 | | 9 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:52 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind.v1 | None | |
| | 282 | | 10 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:56 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas.v1 | None | |
| | 283 | | 11 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:00 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros.v1 | None | |
| | 284 | | 12 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:06 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas.v1 | None | |
| | 285 | | 13 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:12 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas.v1 | None | |
| | 286 | | 14 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:13 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso.v1 | None | |
| | 287 | | 15 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:14 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr.v1 | None | |
| | 288 | | 16 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:15 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax.v1 | None | |
| | 289 | | 17 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:16 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax.v1 | None | |
| | 290 | | 18 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin.v1 | None | |
| | 291 | | 19 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr.v1 | None | |
| | 292 | | 20 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:19 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas.v1 | None | |
| | 293 | +------------------------------------------------------------------------------------------------------------------------------------------------ |
| | 294 | |
| | 295 | }}} |
| | 296 | |
| | 297 | In order undo the scan, run `esgunpublish` with the following flags: |
| | 298 | |
| | 299 | {{{ |
| | 300 | #!sh |
| | 301 | $ ./esgunpublish -i /esg/config/esgcet/esg.ini --skip-gateway --skip-thredds --database-delete --map ~/cordex.txt |
| | 302 | }}} |
| | 303 | |
| | 304 | === Generate a THREDDS catalog === |
| | 305 | |
| | 306 | You can generate the THREDDS catalog with : |
| | 307 | |
| | 308 | {{{ |
| | 309 | #!sh |
| | 310 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds |
| | 311 | INFO 2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml |
| | 312 | INFO 2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml |
| | 313 | INFO 2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml |
| | 314 | INFO 2013-11-19 19:46:31,198 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1.xml |
| | 315 | INFO 2013-11-19 19:46:31,536 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1.xml |
| | 316 | ............................................... |
| | 317 | }}} |
| | 318 | |
| | 319 | |
| | 320 | In order to remove the catalogs from the THREDDS : |
| | 321 | |
| | 322 | {{{ |
| | 323 | #!sh |
| | 324 | $ ./esgunpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-gateway |
| | 325 | }}} |
| | 326 | |
| | 327 | === idx notification === |
| | 328 | |
| | 329 | First, obtain a digital certificate from an ESGF trusted !MyProxy server, and rename it to whatever path you have defined in esg.ini. |
| | 330 | |
| | 331 | [[NoteBox(warn, Remember\, you have to log in a Federation to do it.)]] |
| | 332 | |
| | 333 | {{{ |
| | 334 | #!sh |
| | 335 | $ sudo /usr/local/globus/bin/myproxy-logon -s vesgint-idx.ipsl.jussieu.fr -l blancojc -o ~/.globus/certificate-file |
| | 336 | }}} |
| | 337 | |
| | 338 | Then you can publish the cordex catalog by executing : |
| | 339 | |
| | 340 | {{{ |
| | 341 | #!sh |
| | 342 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --publish |
| | 343 | INFO 2013-11-19 20:01:24,817 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls |
| | 344 | INFO 2013-11-19 20:01:28,678 Result: SUCCESSFUL |
| | 345 | INFO 2013-11-19 20:01:28,678 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss |
| | 346 | INFO 2013-11-19 20:01:32,416 Result: SUCCESSFUL |
| | 347 | INFO 2013-11-19 20:01:32,417 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss |
| | 348 | INFO 2013-11-19 20:01:36,125 Result: SUCCESSFUL |
| | 349 | INFO 2013-11-19 20:01:36,125 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr |
| | 350 | INFO 2013-11-19 20:01:39,964 Result: SUCCESSFUL |
| | 351 | INFO 2013-11-19 20:01:39,965 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc |
| | 352 | }}} |
| | 353 | |
| | 354 | Use `esgunpublish` to delete idx datasets: |
| | 355 | |
| | 356 | {{{ |
| | 357 | #!sh |
| | 358 | $ ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-thredds |
| | 359 | }}} |
| | 360 | |
| | 361 | |
| | 362 | === Running all publication steps === |
| | 363 | |
| | 364 | For convenience, the full publication can be performed with one command. Also, if the arguments are directories rather than a mapfile, the directories will be scanned as if esgscan_directory were run: |
| | 365 | |
| | 366 | {{{ |
| | 367 | #!sh |
| | 368 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds |
| | 369 | INFO 2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml |
| | 370 | INFO 2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml |
| | 371 | INFO 2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml |
| | 372 | }}} |
| | 373 | |
| | 374 | `esgunpublish` will remove the datasets from the idx, THREDDS, and node database in that order: |
| | 375 | |
| | 376 | {{{ |
| | 377 | #!sh |
| | 378 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --database-delete --map ~/cordex.txt |
| | 379 | }}} |
| | 380 | |
| | 381 | === Access files === |
| | 382 | |
| | 383 | Finally, in order to grant access to our files you need to add the line below : |
| | 384 | |
| | 385 | {{{ |
| | 386 | <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/> |
| | 387 | }}} |
| | 388 | |
| | 389 | in your `esgf_policies_local.xml` file : |
| | 390 | |
| | 391 | {{{ |
| | 392 | #!sh |
| | 393 | $ cat /usr/local/apache-tomcat-7.0.47/webapps/esg-orp/WEB-INF/classes/esg/orp/orp/config/esgf_policies_local.xml |
| | 394 | <?xml version="1.0" encoding="UTF-8" standalone="yes"?> |
| | 395 | |
| | 396 | <!-- This file is intended to be edited and maintained by the local Node admnistrators. |
| | 397 | It should contain only policies specific to data served by the local Node. |
| | 398 | It will not be overridden by a software update. --> |
| | 399 | <policies xmlns="http://www.esgf.org/security"> |
| | 400 | <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/> |
| | 401 | <!-- The following statements allow all members of group "CMIP5 Research" or "CMIP5 Commercial" to read any local URL that contains "cmip5". |
| | 402 | Note that the groups "CMIP5 Research" and "CMIP5 Commercial" are administered by PCMDI --> |
| | 403 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="user" action="Read"/> --> |
| | 404 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="user" action="Read"/> --> |
| | 405 | <!-- These statements provide CMIP5 Read access for members of the old gateways --> |
| | 406 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="default" action="Read"/> --> |
| | 407 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="default" action="Read"/> --> |
| | 408 | |
| | 409 | |
| | 410 | <!-- The following statement allows all members of group "MY GROUP" to read any local URL that contains "my_data" |
| | 411 | <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="user" action="Read"/> --> |
| | 412 | |
| | 413 | <!-- The following statement allows members of group "MY GROUP" with role="publisher" to publish local datasets with id containing "my_data" |
| | 414 | <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="publisher" action="Write"/> --> |
| | 415 | |
| | 416 | <!-- The following statements makes resources that contain '.*test.*' freely available for download --> |
| | 417 | <!-- <policy resource=".*test.*" attribute_type="ANY" attribute_value="" action="Read"/> --> |
| | 418 | |
| | 419 | <!-- The following statements makes all resources freely available for download --> |
| | 420 | <!-- <policy resource=".*" attribute_type="ANY" attribute_value="" action="Read"/> --> |
| | 421 | |
| | 422 | </policies> |
| | 423 | }}} |
| | 424 | |
| | 425 | |
| | 426 | |
| | 427 | In production status, you will have to something like this: |
| | 428 | {{{ |
| | 429 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="user" action="Read"/> |
| | 430 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="default" action="Read"/> |
| | 431 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="user" action="Read"/> |
| | 432 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="default" action="Read"/> |
| | 433 | }}} |
| | 434 | |
| | 435 | |
| | 436 | |
