| 1 | = Data Publishing = |
| 2 | |
| 3 | == Configuring a new project for ESGF publication == |
| 4 | |
| 5 | |
| 6 | [[NoteBox(tip, See the [http://esg-pcmdi.llnl.gov/internal/esg-data-node-documentation/introduction-to-esg-data-node-configuration/ ESGF] publication reference for details.)]] |
| 7 | |
| 8 | |
| 9 | In order to publish, you have to configure a text file, `/esg/config/esgcet/esg.ini`. For this propose, we are going to create a new project called `cordex`: |
| 10 | |
| 11 | |
| 12 | {{{ |
| 13 | #!sh |
| 14 | [initialize] |
| 15 | log_level = DEBUG |
| 16 | initial_models_table = /esg/config/esgcet/esgcet_models_table.txt |
| 17 | |
| 18 | |
| 19 | [DEFAULT] |
| 20 | thredds_dataset_roots = |
| 21 | esg_dataroot | /datasets |
| 22 | |
| 23 | project_options = |
| 24 | cordex | CORDEX Output data | 1 |
| 25 | [project:cordex] |
| 26 | # name | category_type | is_mandatory | is_thredds_property | display_order |
| 27 | categories = |
| 28 | project | enum | true | true | 0 |
| 29 | domain | enum | true | true | 1 |
| 30 | institute | enum | true | true | 2 |
| 31 | driving_model | enum | false | true | 3 |
| 32 | experiment | enum | false | true | 4 |
| 33 | ensemble | enum | false | true | 5 |
| 34 | model | enum | false | true | 6 |
| 35 | time_frequency | enum | false | true | 7 |
| 36 | version | enum | false | true | 8 |
| 37 | rcm_model | enum | false | true | 9 |
| 38 | rcm_version | enum | false | true | 10 |
| 39 | description | text | false | false | 99 |
| 40 | category_defaults = |
| 41 | domain | EUR-22 |
| 42 | institute | UCAN |
| 43 | driving_model | ERAINT |
| 44 | ensemble | r1i1p1 |
| 45 | model | WRF331G |
| 46 | time_frequency| mon |
| 47 | dataset_id = cordex.%(domain)s.%(institute)s.%(driving_model)s.%(experiment)s.%(ensemble)s.WRF331G_v02.%(time_frequency)s.%(variable)s |
| 48 | directory_format = /datasets/CORDEX/output/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(rcm_model)s/%(rcm_version)s/%(time_frequency)s/%(variable)s/%(version)s |
| 49 | domain_map = map(project_id,domain : domain_description) |
| 50 | cordex | SAM-44 | South America |
| 51 | cordex | CAM-44 | Central America |
| 52 | cordex | NAM-44 | North America |
| 53 | cordex | EUR-44 | Europe |
| 54 | cordex | EUR-22 | Europe |
| 55 | cordex | AFR-44 | Africa |
| 56 | cordex | WAS-44 | West Asia |
| 57 | cordex | EAS-44 | East Asia |
| 58 | cordex | CAS-44 | Central Asia |
| 59 | cordex | AUS-44 | Australasia |
| 60 | cordex | ANT-44 | Antarctica |
| 61 | cordex | ARC-44 | The Arctic |
| 62 | cordex | MED-44 | HYMEX Mediterranean |
| 63 | cordex | EUR-11 | High-res. Europe |
| 64 | cordex | SAM-44i | South America |
| 65 | cordex | CAM-44i | Central America |
| 66 | cordex | NAM-44i | North America |
| 67 | cordex | EUR-44i | Europe |
| 68 | cordex | AFR-44i | Africa |
| 69 | cordex | WAS-44i | West Asia |
| 70 | cordex | EAS-44i | East Asia |
| 71 | cordex | CAS-44i | Central Asia |
| 72 | cordex | AUS-44i | Australasia |
| 73 | cordex | ANT-44i | Antarctica |
| 74 | cordex | ARC-44i | The Arctic |
| 75 | cordex | MED-44i | HYMEX Mediterranean |
| 76 | cordex | EUR-11i | High-res. Europe |
| 77 | cordex | MNA-44 | Middle East and North Africa |
| 78 | cordex | MNA-44i | Middle East and North Africa |
| 79 | cordex | MNA-22 | Middle East and North Africa |
| 80 | cordex | MNA-22i | Middle East and North Africa |
| 81 | domain_options = SAM-44,CAM-44,NAM-44,EUR-44,EUR-22,EUR-44i,AFR-44,AFR-44i,WAS-44,EAS-44,CAS-44,AUS-44,ANT-44,ARC-44,MED-44,EUR-11,SAM-44i,CAM-44i,NAM-44i,EUR-44i,AFR-44i,WAS-44i,EAS-44i,CAS-44i,AUS-44i,ANT-44i,ARC-44i,MED-44i,EUR-11i,MNA-44,MNA-44i,MNA-22,MNA-22i |
| 82 | driving_model_options = ERAINT, ECMWF-ERAINT, CCCma-CanESM2, CNRM-CERFACS-CNRM-CM5, ICHEC-EC-EARTH, MIROC-MIROC5, MOHC-HadGEM2-ES, MPI-M-MPI-ESM-LR, NCC-NorESM1-M, NOAA-GFDL-GFDL-ESM2M, IPSL-IPSL-CM5A-MR |
| 83 | ensemble_options = r1i1p1, r12i1p1, r0i0p0 |
| 84 | experiment_options = |
| 85 | cordex | evaluation | no description |
| 86 | cordex | historical | no description |
| 87 | cordex | rcp4 | no description |
| 88 | cordex | rcp26 | no description |
| 89 | cordex | rcp45 | no description |
| 90 | cordex | rcp85 | no description |
| 91 | institute_map = map(project_id,model : institute) |
| 92 | cordex | WRF331G | UCAN |
| 93 | institute_options = UCAN |
| 94 | las_configure = false |
| 95 | las_time_delta_map = map(time_frequency : las_time_delta) |
| 96 | mon | 1 month |
| 97 | day | 1 day |
| 98 | fx | fixed |
| 99 | sem | semi |
| 100 | maps = institute_map, las_time_delta_map, domain_map |
| 101 | model_options = WRF331G |
| 102 | parent_id = wdcc2.cordex |
| 103 | project_handler_name = basic_builtin |
| 104 | rcm_model_options = UCAN-WRF331G |
| 105 | rcm_version_options = v02 |
| 106 | thredds_exclude_variables = a, a_bnds, alev1, alevel, alevhalf, alt40, b, b_bnds, basin, bnds, bounds_lat, bounds_lon, dbze, depth, depth0m, depth100m, depth_bnds, geo_region, height, height10m, height2m, Lambert_Conformal, lat, lat_bnds, lat_bounds, latitude, latitude_bnds, layer, lev, lev_bnds, location, lon, lon_bnds, lon_bounds, longitude, longitude_bnds, olayer100m, olevel, oline, p0, p220, p500, p560, p700, p840, plev, plev3, plev7, plev8, plev_bnds, plevs, pressure1, region, rho, rlat, rotated_pole, rlon, scatratio, sdepth, sdepth1, sza5, tau, tau_bnds, time, time1, time2, time_bnds, vegtype, x, y |
| 107 | time_frequency_options = day,fx,mon,sem,3hr,6hr |
| 108 | variable_locate = ps,ps_ | vas,vas_ | uas,uas_ |tasmin,tasmin_|tasmax,tasmax_| tas,tas_ | sfcWindmax,sfcWindmax_ | sfcWind,sfcWind_ |rsds,rsds_|rlds,rlds_| pr,pr_ | prc,prc_| mrso,mrso_| mrros,mrros_|huss,huss_| hfss,hfss_|hfls,hfls_ |
| 109 | variable_per_file = true |
| 110 | version_options = 20131108 |
| 111 | }}} |
| 112 | |
| 113 | Therefore, if you use the above configuration file, you will have to create a tree directory like this: |
| 114 | |
| 115 | {{{ |
| 116 | |
| 117 | [esgf@data datasets]# tree |
| 118 | . |
| 119 | `-- CORDEX |
| 120 | `-- output |
| 121 | `-- EUR-22 |
| 122 | `-- UCAN |
| 123 | `-- ECMWF-ERAINT |
| 124 | `-- evaluation |
| 125 | `-- r1i1p1 |
| 126 | `-- UCAN-WRF311G |
| 127 | `-- v02 |
| 128 | |-- 3hr |
| 129 | | |-- hfls |
| 130 | | | `-- 20131108 |
| 131 | | | |-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 132 | | | `-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 133 | | |-- hfss |
| 134 | | | `-- 20131108 |
| 135 | | | |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 136 | | | |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc |
| 137 | | | `-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 138 | | |-- huss |
| 139 | | | `-- 20131108 |
| 140 | | | |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 141 | | | |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc |
| 142 | | | `-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 143 | | |-- pr |
| 144 | | | `-- 20131108 |
| 145 | | | |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 146 | | | `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 147 | | |-- prc |
| 148 | | | `-- 20131108 |
| 149 | | | |-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 150 | | | `-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 151 | | |-- ps |
| 152 | | | `-- 20131108 |
| 153 | | | |-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 154 | | | `-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 155 | | |-- rlds |
| 156 | | | `-- 20131108 |
| 157 | | | |-- rlds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 158 | | | |-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc |
| 159 | | | `-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 160 | | |-- sfcWind |
| 161 | | | `-- 20131108 |
| 162 | | | |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 163 | | | |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc |
| 164 | | | |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc |
| 165 | | | `-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 166 | | `-- tas |
| 167 | | `-- 20131108 |
| 168 | | |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc |
| 169 | | `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc |
| 170 | |-- 6hr |
| 171 | | |-- mrros |
| 172 | | | `-- 20131108 |
| 173 | | | |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc |
| 174 | | | |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc |
| 175 | | | `-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc |
| 176 | | |-- uas |
| 177 | | | `-- 20131108 |
| 178 | | | |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc |
| 179 | | | |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc |
| 180 | | | `-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc |
| 181 | | `-- vas |
| 182 | | `-- 20131108 |
| 183 | | |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc |
| 184 | | |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc |
| 185 | | `-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc |
| 186 | |-- day |
| 187 | | |-- mrso |
| 188 | | | `-- 20131108 |
| 189 | | | |-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| 190 | | | `-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| 191 | | |-- pr |
| 192 | | | `-- 20131108 |
| 193 | | | |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| 194 | | | `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| 195 | | |-- sfcWindmax |
| 196 | | | `-- 20131108 |
| 197 | | | |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| 198 | | | |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20010101-20051231.nc |
| 199 | | | `-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| 200 | | |-- tasmax |
| 201 | | | `-- 20131108 |
| 202 | | | |-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| 203 | | | `-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| 204 | | `-- tasmin |
| 205 | | `-- 20131108 |
| 206 | | |-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc |
| 207 | | `-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc |
| 208 | `-- mon |
| 209 | |-- pr |
| 210 | | `-- 20131108 |
| 211 | | |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc |
| 212 | | `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc |
| 213 | `-- tas |
| 214 | `-- 20131108 |
| 215 | |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc |
| 216 | |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19910101-20001231.nc |
| 217 | `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc |
| 218 | |
| 219 | |
| 220 | }}} |
| 221 | |
| 222 | |
| 223 | Then you have to add the project name to the `esgcet_models_table.txt` file |
| 224 | |
| 225 | {{{ |
| 226 | #!sh |
| 227 | $ echo " cordex | WRF331G | http://meteo.unican.es | UNICAN WRF3.3.1 Model version, 2.0" >> /esg/config/esgcet/esgcet_models_table.txt |
| 228 | }}} |
| 229 | |
| 230 | |
| 231 | After modifying `esgcet_models_table.txt` and `esg.ini` files, you have to update the data base by executing : |
| 232 | {{{ |
| 233 | #!sh |
| 234 | $ cd /usr/local/uvcdat/1.4.0/bin/ |
| 235 | $ ./esginitialize -i /esg/config/esgcet/esg.ini -c |
| 236 | }}} |
| 237 | |
| 238 | * `./esginitialize -d 0`: To remove all tables |
| 239 | |
| 240 | * `./esginitialize -c` : Upgrade the database schema to the latest version, and initialize projects, models, experiments, and standard names from configuration files. |
| 241 | |
| 242 | == Using the ESGF Publisher == |
| 243 | |
| 244 | |
| 245 | This takes place in three steps: |
| 246 | |
| 247 | * Scan each file for metadata and save the metadata in the node database. (This is in contrast to running `esgscan_directory`, which just scans the directory structure.) |
| 248 | |
| 249 | * Generate a THREDDS catalog based on the scanned information. THREDDS is a data and metadata server used by ESGF. |
| 250 | |
| 251 | * Notify the idx that one or more catalogs have been generated. |
| 252 | |
| 253 | === File Scan Phase === |
| 254 | |
| 255 | In order to scan the cordex files for metadata, run `esgscan_directory` to generate a mapfile and after that run `esgpublish` with input from a mapfile: |
| 256 | {{{ |
| 257 | #!sh |
| 258 | $ whoami |
| 259 | esgf |
| 260 | $ cd /usr/local/uvcdat/1.4.0/bin |
| 261 | $ ./esgscan_directory -i /esg/config/esgcet/esg.ini --project cordex -o ~/cordex.txt /datasets/CORDEX |
| 262 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt |
| 263 | }}} |
| 264 | |
| 265 | In order to check that the datasets were entered correctly into the database, list the datasets for cordex project: |
| 266 | |
| 267 | {{{ |
| 268 | #!sh |
| 269 | $ ./esglist_datasets -i /esg/config/esgcet/esg.ini cordex |
| 270 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |
| 271 | | id | name | project | model | experiment | run_name | offline | master_gateway | domain | institute | publish_time | publish_status | parent | version | version_name | comment | |
| 272 | +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ |
| 273 | | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:21 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1 | None | |
| 274 | | 2 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:25 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1 | None | |
| 275 | | 3 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:29 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1 | None | |
| 276 | | 4 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:33 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1 | None | |
| 277 | | 5 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:37 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1 | None | |
| 278 | | 6 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:40 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps.v1 | None | |
| 279 | | 7 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:44 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds.v1 | None | |
| 280 | | 8 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:48 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds.v1 | None | |
| 281 | | 9 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:52 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind.v1 | None | |
| 282 | | 10 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:42:56 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas.v1 | None | |
| 283 | | 11 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:00 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros.v1 | None | |
| 284 | | 12 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:06 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas.v1 | None | |
| 285 | | 13 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:12 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas.v1 | None | |
| 286 | | 14 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:13 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso.v1 | None | |
| 287 | | 15 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:14 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr.v1 | None | |
| 288 | | 16 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:15 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax.v1 | None | |
| 289 | | 17 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:16 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax.v1 | None | |
| 290 | | 18 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin.v1 | None | |
| 291 | | 19 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr.v1 | None | |
| 292 | | 20 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas | cordex | WRF331G | evaluation | None | False | None | EUR-22 | UCAN | 2013-11-18 19:43:19 | PUBLISH_DATASET | ROOT | 1 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas.v1 | None | |
| 293 | +------------------------------------------------------------------------------------------------------------------------------------------------ |
| 294 | |
| 295 | }}} |
| 296 | |
| 297 | In order undo the scan, run `esgunpublish` with the following flags: |
| 298 | |
| 299 | {{{ |
| 300 | #!sh |
| 301 | $ ./esgunpublish -i /esg/config/esgcet/esg.ini --skip-gateway --skip-thredds --database-delete --map ~/cordex.txt |
| 302 | }}} |
| 303 | |
| 304 | === Generate a THREDDS catalog === |
| 305 | |
| 306 | You can generate the THREDDS catalog with : |
| 307 | |
| 308 | {{{ |
| 309 | #!sh |
| 310 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds |
| 311 | INFO 2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml |
| 312 | INFO 2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml |
| 313 | INFO 2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml |
| 314 | INFO 2013-11-19 19:46:31,198 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1.xml |
| 315 | INFO 2013-11-19 19:46:31,536 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1.xml |
| 316 | ............................................... |
| 317 | }}} |
| 318 | |
| 319 | |
| 320 | In order to remove the catalogs from the THREDDS : |
| 321 | |
| 322 | {{{ |
| 323 | #!sh |
| 324 | $ ./esgunpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-gateway |
| 325 | }}} |
| 326 | |
| 327 | === idx notification === |
| 328 | |
| 329 | First, obtain a digital certificate from an ESGF trusted !MyProxy server, and rename it to whatever path you have defined in esg.ini. |
| 330 | |
| 331 | [[NoteBox(warn, Remember\, you have to log in a Federation to do it.)]] |
| 332 | |
| 333 | {{{ |
| 334 | #!sh |
| 335 | $ sudo /usr/local/globus/bin/myproxy-logon -s vesgint-idx.ipsl.jussieu.fr -l blancojc -o ~/.globus/certificate-file |
| 336 | }}} |
| 337 | |
| 338 | Then you can publish the cordex catalog by executing : |
| 339 | |
| 340 | {{{ |
| 341 | #!sh |
| 342 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --publish |
| 343 | INFO 2013-11-19 20:01:24,817 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls |
| 344 | INFO 2013-11-19 20:01:28,678 Result: SUCCESSFUL |
| 345 | INFO 2013-11-19 20:01:28,678 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss |
| 346 | INFO 2013-11-19 20:01:32,416 Result: SUCCESSFUL |
| 347 | INFO 2013-11-19 20:01:32,417 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss |
| 348 | INFO 2013-11-19 20:01:36,125 Result: SUCCESSFUL |
| 349 | INFO 2013-11-19 20:01:36,125 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr |
| 350 | INFO 2013-11-19 20:01:39,964 Result: SUCCESSFUL |
| 351 | INFO 2013-11-19 20:01:39,965 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc |
| 352 | }}} |
| 353 | |
| 354 | Use `esgunpublish` to delete idx datasets: |
| 355 | |
| 356 | {{{ |
| 357 | #!sh |
| 358 | $ ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-thredds |
| 359 | }}} |
| 360 | |
| 361 | |
| 362 | === Running all publication steps === |
| 363 | |
| 364 | For convenience, the full publication can be performed with one command. Also, if the arguments are directories rather than a mapfile, the directories will be scanned as if esgscan_directory were run: |
| 365 | |
| 366 | {{{ |
| 367 | #!sh |
| 368 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds |
| 369 | INFO 2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml |
| 370 | INFO 2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml |
| 371 | INFO 2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml |
| 372 | }}} |
| 373 | |
| 374 | `esgunpublish` will remove the datasets from the idx, THREDDS, and node database in that order: |
| 375 | |
| 376 | {{{ |
| 377 | #!sh |
| 378 | $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --database-delete --map ~/cordex.txt |
| 379 | }}} |
| 380 | |
| 381 | === Access files === |
| 382 | |
| 383 | Finally, in order to grant access to our files you need to add the line below : |
| 384 | |
| 385 | {{{ |
| 386 | <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/> |
| 387 | }}} |
| 388 | |
| 389 | in your `esgf_policies_local.xml` file : |
| 390 | |
| 391 | {{{ |
| 392 | #!sh |
| 393 | $ cat /usr/local/apache-tomcat-7.0.47/webapps/esg-orp/WEB-INF/classes/esg/orp/orp/config/esgf_policies_local.xml |
| 394 | <?xml version="1.0" encoding="UTF-8" standalone="yes"?> |
| 395 | |
| 396 | <!-- This file is intended to be edited and maintained by the local Node admnistrators. |
| 397 | It should contain only policies specific to data served by the local Node. |
| 398 | It will not be overridden by a software update. --> |
| 399 | <policies xmlns="http://www.esgf.org/security"> |
| 400 | <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/> |
| 401 | <!-- The following statements allow all members of group "CMIP5 Research" or "CMIP5 Commercial" to read any local URL that contains "cmip5". |
| 402 | Note that the groups "CMIP5 Research" and "CMIP5 Commercial" are administered by PCMDI --> |
| 403 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="user" action="Read"/> --> |
| 404 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="user" action="Read"/> --> |
| 405 | <!-- These statements provide CMIP5 Read access for members of the old gateways --> |
| 406 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="default" action="Read"/> --> |
| 407 | <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="default" action="Read"/> --> |
| 408 | |
| 409 | |
| 410 | <!-- The following statement allows all members of group "MY GROUP" to read any local URL that contains "my_data" |
| 411 | <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="user" action="Read"/> --> |
| 412 | |
| 413 | <!-- The following statement allows members of group "MY GROUP" with role="publisher" to publish local datasets with id containing "my_data" |
| 414 | <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="publisher" action="Write"/> --> |
| 415 | |
| 416 | <!-- The following statements makes resources that contain '.*test.*' freely available for download --> |
| 417 | <!-- <policy resource=".*test.*" attribute_type="ANY" attribute_value="" action="Read"/> --> |
| 418 | |
| 419 | <!-- The following statements makes all resources freely available for download --> |
| 420 | <!-- <policy resource=".*" attribute_type="ANY" attribute_value="" action="Read"/> --> |
| 421 | |
| 422 | </policies> |
| 423 | }}} |
| 424 | |
| 425 | |
| 426 | |
| 427 | In production status, you will have to something like this: |
| 428 | {{{ |
| 429 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="user" action="Read"/> |
| 430 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="default" action="Read"/> |
| 431 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="user" action="Read"/> |
| 432 | <policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="default" action="Read"/> |
| 433 | }}} |
| 434 | |
| 435 | |
| 436 | |