wiki:ESGFCordexDataPublication

Version 5 (modified by carlos, 7 years ago) (diff)

--

Data Publishing

Configuring CORDEX project for ESGF publication

In order to publish, you have to configure a text file, /esg/config/esgcet/esg.ini. For this propose, we are going to create a new project called cordex:

[project:cordex]
categories =
        project          | enum | true | true | 0
        domain           | enum | true | true | 1
        institute        | enum | true | true | 2
        product          | enum | true | true | 3
        driving_model    | enum | false | true | 4
        experiment       | enum | false | true | 5
        ensemble         | enum | false | true | 6
        model            | enum | false | true | 7
        time_frequency   | enum | false | true | 8
        version          | enum | false | true | 9
        rcm_model        | enum | false | true | 10
        rcm_version      | enum | false | true | 11
        description      | text | false | false | 99
category_defaults =
        domain | EUR-44
        institute | UCAN
        driving_model | ECMWF-ERAINT
        ensemble | r1i1p1
        time_frequency| mon
        experiment | evaluation
        model| WRF331G
        product | output
model_map = map(project,rcm_model : model)
        cordex| AUTH-LHTEE-WRF321B| WRF321B
        cordex| AUTH-Met-WRF331A| WRF331A
        cordex| AWI-HIRHAM5| HIRHAM5
        cordex| BCCR-WRF331C| WRF331C
        cordex| CCCma-CanRCM4| CanRCM4
        cordex| CHMI-ALADIN52| ALADIN52
        cordex| CLMcom-CCLM4-8-17| CCLM4-8-17
        cordex| CNRM-ALADIN52| ALADIN52
        cordex| CNRM-ARPEGE52| ARPEGE52
        cordex| CRP-GL-WRF331A| WRF331A
        cordex| CUNI-RegCM4-2| RegCM4-2
        cordex| DHMZ-RegCM4-2| RegCM4-2
        cordex| DMI-HIRHAM5| HIRHAM5
        cordex| ENEA-RegCM4-3| RegCM4-3
        cordex| HMS-ALADIN52| ALADIN52
        cordex| ICTP-RegCM4-3| RegCM4-3
        cordex| IDL-WRF331D| WRF331D
        cordex| IPSL-INERIS-WRF331F| WRF331F
        cordex| KNMI-RACMO21P| RACMO21P
        cordex| KNMI-RACMO22T| RACMO22T
        cordex| KNMI-RACMO22E| RACMO22E
        cordex| MIUB-WRF331A| WRF331A
        cordex| MOHC-HadGEM3-RA| HadGEM3-RA
        cordex| MOHC-HadRM3P| HadRM3P
        cordex| MPI-CSC-REMO2009| REMO2009
        cordex| NUIM-WRF331F| WRF331F
        cordex| SMHI-RCA4| RCA4
        cordex| SMHI-RCA4-SN| RCA4-SN
        cordex| SMHI-RCAO| RCAO
        cordex| SMHI-RCAO-SN| RCAO-SN
        cordex| UCAN-WRF331G| WRF331G
        cordex| UCAN-WRF350I| WRF350I
        cordex| UCLM-PROMES| PROMES
        cordex| UHOH-WRF331H| WRF331H
        cordex| UQAM-CRCM5| CRCM5
domain_map = map(project_id,domain : domain_description)
        cordex | SAM-44 | South America
        cordex | CAM-44 | Central America
        cordex | NAM-44 | North America
        cordex | EUR-44 | Europe
        cordex | EUR-22 | Europe
        cordex | AFR-44 | Africa
        cordex | WAS-44 | West Asia
        cordex | EAS-44 | East Asia
        cordex | CAS-44 | Central Asia
        cordex | AUS-44 | Australasia
        cordex | ANT-44 | Antarctica
        cordex | ARC-44 | The Arctic
        cordex | MED-44 | HYMEX Mediterranean
        cordex | EUR-11 | High-res. Europe
        cordex | SAM-44i | South America
        cordex | CAM-44i | Central America
        cordex | NAM-44i | North America
        cordex | EUR-44i | Europe
        cordex | AFR-44i | Africa
        cordex | WAS-44i | West Asia
        cordex | EAS-44i | East Asia
        cordex | CAS-44i | Central Asia
        cordex | AUS-44i | Australasia
        cordex | ANT-44i | Antarctica
        cordex | ARC-44i | The Arctic
        cordex | MED-44i | HYMEX Mediterranean
        cordex | EUR-11i | High-res. Europe
        cordex | MNA-44  | Middle East and North Africa
        cordex | MNA-44i | Middle East and North Africa
        cordex | MNA-22  | Middle East and North Africa
        cordex | MNA-22i | Middle East and North Africa
domain_options = SAM-44,CAM-44,NAM-44,EUR-44,EUR-22,EUR-44i,AFR-44,AFR-44i,WAS-44,EAS-44,CAS-44,AUS-44,ANT-44,ARC-44,MED-44,EUR-11,SAM-44i,CAM-44i,NAM-44i,EUR-44i,AFR-44i,WAS-44i,EAS-44i,CAS-44i,AUS-44i,ANT-44i,ARC-44i,MED-44i,EUR-11i,MNA-44,MNA-44i,MNA-22,MNA-22i
driving_model_options = ERAINT, ECMWF-ERAINT, CCCma-CanESM2, CNRM-CERFACS-CNRM-CM5, ICHEC-EC-EARTH, MIROC-MIROC5, MOHC-HadGEM2-ES, MPI-M-MPI-ESM-LR, NCC-NorESM1-M, NOAA-GFDL-GFDL-ESM2M, IPSL-IPSL-CM5A-MR
ensemble_options = r1i1p1, r12i1p1, r0i0p0
product_options = output1, output2, output
experiment_options =
        cordex | evaluation | no description
        cordex | historical | no description
        cordex | rcp4 | no description
        cordex | rcp26 | no description
        cordex | rcp45 | no description
        cordex | rcp85 | no description
institute_map = map(project_id,model : institute)
        cordex | WRF331G | UCAN
institute_options = UCAN
las_configure = false
las_time_delta_map = map(time_frequency : las_time_delta)
        mon     | 1 month
        day     | 1 day
        fx      | fixed
        sem     | semi
maps = institute_map, las_time_delta_map, domain_map
model_options = WRF331G
parent_id = wdcc2.cordex
project_handler_name = basic_builtin
rcm_model_options = UCAN-WRF331G
rcm_version_options = v01,v02
thredds_exclude_variables = a, a_bnds, alev1, alevel, alevhalf, alt40, b, b_bnds, basin, bnds, bounds_lat, bounds_lon, dbze, depth, depth0m, depth100m, depth_bnds, geo_region, height, height10m, height2m, Lambert_Conformal, lat, lat_bnds, lat_bounds, latitude, latitude_bnds, layer, lev, lev_bnds, location, lon, lon_bnds, lon_bounds, longitude, longitude_bnds, olayer100m, olevel, oline, p0, p220, p500, p560, p700, p840, plev, plev3, plev7, plev8, plev_bnds, plevs, pressure1, region, rho, rlat, rotated_pole, rlon, scatratio, sdepth, sdepth1, sza5, tau, tau_bnds, time, time1, time2, time_bnds, vegtype, x, y, Rotated_Pole, heightv
time_frequency_options = day,fx,mon,sem,3hr,6hr
variable_locate = clivi, clivi_|  clt, clt_|  evspsbl, evspsbl_ | hfls , hfls_ | hfss , hfss_ | hus850 , hus850_|  huss ,huss_|  mrros ,  mrros_ | prc , prc_ | pr, pr_|  prsn , prsn_| prw , prw_ | ps , ps_ |psl, psl_| rlds, rlds_|  rlus ,  rlus_ |rlut , rlut_|  rsds , rsds_ | rsdt , rsdt_| rsus, rsus_|  rsut , rsut_ |  sfcWind , sfcWind_ | sfcWindmax , sfcWindmax_| snd , snd_|snm ,snm_ | snw , snw_| ta200 , ta200_ |ta500 , ta500_ | ta850, ta850_ | tas, tas_|  tasmax, tasmax_ | tasmin , tasmin_| ts, ts_|  ua200 , ua200_|  ua500 , ua500_ | ua850 , ua850_| uas , uas_| va200, va200_ |va500, va500_ | va850, va850_ | vas, vas_| zg200, zg200_ |zg500, zg500_ | zmla ,  zmla_
variable_per_file = true
version_options = 20140328
model= WRF331G

dataset_id = cordex.%(product)s.%(domain)s.%(institute)s.%(driving_model)s.%(experiment)s.%(ensemble)s.%(model)s.%(rcm_version)s.%(time_frequency)s.%(variable)s
directory_format = /datasets/cordex-noncommercial/cordex/%(product)s/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(rcm_model)s/%(rcm_version)s/%(time_frequency)s/%(variable)s/v%(version)s

Therefore, if you use the above configuration file, you will have to create a tree directory like this:

[root@data ~]# tree /datasets/
/datasets/
`-- cordex-noncommercial
    `-- cordex
        `-- output
            |-- EUR-22
            |   `-- UCAN
            |       `-- ECMWF-ERAINT
            |           `-- evaluation
            |               `-- r1i1p1
            |                   `-- UCAN-WRF331G
            |                       `-- v02
            |                           |-- 3hr
            |                           |   |-- clivi
            |                           |   |   `-- v20140328
            |                           |   |-- clt
            |                           |   |   `-- v20140328
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19810101-19811231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19810101-19811231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19820101-19821231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19820101-19821231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19830101-19831231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19830101-19831231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19840101-19841231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19840101-19841231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19850101-19851231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19850101-19851231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19860101-19861231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19860101-19861231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19870101-19871231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19870101-19871231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19880101-19881231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19880101-19881231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19890101-19891231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19890101-19891231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19900101-19901231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19900101-19901231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19910101-19911231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19910101-19911231.nc
            |                           |   |       |-- clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19920101-19921231.nc -> /vols/seal/oceano/gmeteo/DATA/UC/eurocordex/01_EuroCORDEX_INTERIM_022/clt_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19920101-19921231.nc

Then you have to add the project name to the esgcet_models_table.txt file

$ echo "   cordex | WRF331G | http://meteo.unican.es | UNICAN WRF3.3.1 Model version, 2.0" >> /esg/config/esgcet/esgcet_models_table.txt 

After modifying esgcet_models_table.txt and esg.ini files, you have to update the data base by executing :

$ cd /usr/local/uvcdat/1.4.0/bin/
$ ./esginitialize -i /esg/config/esgcet/esg.ini -c
  • ./esginitialize -d 0: To remove all tables
  • ./esginitialize -c : Upgrade the database schema to the latest version, and initialize projects, models, experiments, and standard names from configuration files.

Using the ESGF Publisher

This takes place in three steps:

  • Scan each file for metadata and save the metadata in the node database. (This is in contrast to running esgscan_directory, which just scans the directory structure.)
  • Generate a THREDDS catalog based on the scanned information. THREDDS is a data and metadata server used by ESGF.
  • Notify the idx that one or more catalogs have been generated.

File Scan Phase

In order to scan the cordex files for metadata, run esgscan_directory to generate a mapfile and after that run esgpublish with input from a mapfile:

$ whoami 
esgf
$ cd /usr/local/uvcdat/1.4.0/bin
$ ./esgscan_directory -i /esg/config/esgcet/esg.ini --project cordex -o ~/cordex.txt /datasets/CORDEX
$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt 

In order to check that the datasets were entered correctly into the database, list the datasets for cordex project:

$ ./esglist_datasets -i /esg/config/esgcet/esg.ini cordex
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| id | name                                                                         | project | model   | experiment | run_name | offline | master_gateway | domain | institute | publish_time        | publish_status  | parent | version | version_name                                                                    | comment |
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 1  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:21 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1       | None    |
| 2  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:25 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1       | None    |
| 3  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:29 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1       | None    |
| 4  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:33 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1         | None    |
| 5  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:37 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1        | None    |
| 6  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:40 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps.v1         | None    |
| 7  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:44 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds.v1       | None    |
| 8  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:48 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds.v1       | None    |
| 9  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind    | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:52 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind.v1    | None    |
| 10 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:56 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas.v1        | None    |
| 11 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros      | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:00 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros.v1      | None    |
| 12 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:06 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas.v1        | None    |
| 13 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:12 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas.v1        | None    |
| 14 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:13 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso.v1       | None    |
| 15 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:14 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr.v1         | None    |
| 16 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:15 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax.v1 | None    |
| 17 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax     | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:16 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax.v1     | None    |
| 18 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin     | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin.v1     | None    |
| 19 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr.v1         | None    |
| 20 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:19 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas.v1        | None    |
+------------------------------------------------------------------------------------------------------------------------------------------------

In order undo the scan, run esgunpublish with the following flags:

$ ./esgunpublish  -i /esg/config/esgcet/esg.ini --skip-gateway --skip-thredds --database-delete --map ~/cordex.txt

Generate a THREDDS catalog

You can generate the THREDDS catalog with :

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds
INFO       2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml
INFO       2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml
INFO       2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml
INFO       2013-11-19 19:46:31,198 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1.xml
INFO       2013-11-19 19:46:31,536 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1.xml
...............................................

In order to remove the catalogs from the THREDDS :

$ ./esgunpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-gateway

idx notification

First, obtain a digital certificate from an ESGF trusted MyProxy server, and rename it to whatever path you have defined in esg.ini.

Remember, you have to log in a Federation to do it.

$ sudo /usr/local/globus/bin/myproxy-logon -s vesgint-idx.ipsl.jussieu.fr -l blancojc -o ~/.globus/certificate-file

Then you can publish the cordex catalog by executing :

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --publish
INFO       2013-11-19 20:01:24,817 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls
INFO       2013-11-19 20:01:28,678   Result: SUCCESSFUL
INFO       2013-11-19 20:01:28,678 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss
INFO       2013-11-19 20:01:32,416   Result: SUCCESSFUL
INFO       2013-11-19 20:01:32,417 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss
INFO       2013-11-19 20:01:36,125   Result: SUCCESSFUL
INFO       2013-11-19 20:01:36,125 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr
INFO       2013-11-19 20:01:39,964   Result: SUCCESSFUL
INFO       2013-11-19 20:01:39,965 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc

Use esgunpublish to delete idx datasets:

$ ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-thredds

Running all publication steps

For convenience, the full publication can be performed with one command. Also, if the arguments are directories rather than a mapfile, the directories will be scanned as if esgscan_directory were run:

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds
INFO       2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml
INFO       2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml
INFO       2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml

esgunpublish will remove the datasets from the idx, THREDDS, and node database in that order:

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --database-delete --map ~/cordex.txt

Access files

Finally, in order to grant access to our files you need to add the line below :

<policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/>

in your esgf_policies_local.xml file :

$ cat /usr/local/apache-tomcat-7.0.47/webapps/esg-orp/WEB-INF/classes/esg/orp/orp/config/esgf_policies_local.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>

<!-- This file is intended to be edited and maintained by the local Node admnistrators.
     It should contain only policies specific to data served by the local Node.
     It will not be overridden by a software update. -->
<policies xmlns="http://www.esgf.org/security">
     <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/>
    <!-- The following statements allow all members of group "CMIP5 Research" or "CMIP5 Commercial" to read any local URL that contains "cmip5".
         Note that the groups "CMIP5 Research" and "CMIP5 Commercial" are administered by PCMDI -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="user" action="Read"/> -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="user" action="Read"/> -->
    <!-- These statements provide CMIP5 Read access for members of the old gateways -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="default" action="Read"/> -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="default" action="Read"/> -->
    

    <!-- The following statement allows all members of group "MY GROUP" to read any local URL that contains "my_data"
    <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="user" action="Read"/> -->
    
    <!-- The following statement allows members of group "MY GROUP" with role="publisher" to publish local datasets with id containing "my_data" 
    <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="publisher" action="Write"/> -->
    
    <!-- The following statements makes resources that contain '.*test.*' freely available for download -->
    <!-- <policy resource=".*test.*" attribute_type="ANY" attribute_value="" action="Read"/> -->
    
    <!-- The following statements makes all resources freely available for download -->
    <!-- <policy resource=".*" attribute_type="ANY" attribute_value="" action="Read"/> -->

</policies>

In production status, you will have to something like this:

<policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="user" action="Read"/>
<policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="default" action="Read"/>
<policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="user" action="Read"/>
<policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="default" action="Read"/>

v20140328

Attachments (2)

Download all attachments as: .zip