WikiPrint - from Polar Technologies

Data Publishing

Configuring a new project for ESGF publication

See the ?ESGF publication reference for details.

In order to publish, you have to configure a text file, /esg/config/esgcet/esg.ini. For this propose, we are going to create a new project called cordex:

[initialize]
log_level = DEBUG
initial_models_table = /esg/config/esgcet/esgcet_models_table.txt


[DEFAULT]
thredds_dataset_roots =
        esg_dataroot | /datasets

project_options =
        cordex | CORDEX Output data | 1
[project:cordex]
#       name | category_type | is_mandatory | is_thredds_property | display_order
categories =
        project          | enum | true | true | 0
        domain           | enum | true | true | 1
        institute        | enum | true | true | 2
        driving_model    | enum | false | true | 3
        experiment       | enum | false | true | 4
        ensemble         | enum | false | true | 5
        model            | enum | false | true | 6
        time_frequency   | enum | false | true | 7
        version          | enum | false | true | 8
        rcm_model        | enum | false | true | 9
        rcm_version      | enum | false | true | 10
        description      | text | false | false | 99
category_defaults =
        domain | EUR-22
        institute | UCAN
        driving_model | ERAINT
        ensemble | r1i1p1
        model | WRF331G
        time_frequency| mon
dataset_id = cordex.%(domain)s.%(institute)s.%(driving_model)s.%(experiment)s.%(ensemble)s.WRF331G_v02.%(time_frequency)s.%(variable)s
directory_format = /datasets/CORDEX/output/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(rcm_model)s/%(rcm_version)s/%(time_frequency)s/%(variable)s/%(version)s
domain_map = map(project_id,domain : domain_description)
        cordex | SAM-44 | South America
        cordex | CAM-44 | Central America
        cordex | NAM-44 | North America
        cordex | EUR-44 | Europe
        cordex | EUR-22 | Europe
        cordex | AFR-44 | Africa
        cordex | WAS-44 | West Asia
        cordex | EAS-44 | East Asia
        cordex | CAS-44 | Central Asia
        cordex | AUS-44 | Australasia
        cordex | ANT-44 | Antarctica
        cordex | ARC-44 | The Arctic
        cordex | MED-44 | HYMEX Mediterranean
        cordex | EUR-11 | High-res. Europe
        cordex | SAM-44i | South America
        cordex | CAM-44i | Central America
        cordex | NAM-44i | North America
        cordex | EUR-44i | Europe
        cordex | AFR-44i | Africa
        cordex | WAS-44i | West Asia
        cordex | EAS-44i | East Asia
        cordex | CAS-44i | Central Asia
        cordex | AUS-44i | Australasia
        cordex | ANT-44i | Antarctica
        cordex | ARC-44i | The Arctic
        cordex | MED-44i | HYMEX Mediterranean
        cordex | EUR-11i | High-res. Europe
        cordex | MNA-44  | Middle East and North Africa
        cordex | MNA-44i | Middle East and North Africa
        cordex | MNA-22  | Middle East and North Africa
        cordex | MNA-22i | Middle East and North Africa
domain_options = SAM-44,CAM-44,NAM-44,EUR-44,EUR-22,EUR-44i,AFR-44,AFR-44i,WAS-44,EAS-44,CAS-44,AUS-44,ANT-44,ARC-44,MED-44,EUR-11,SAM-44i,CAM-44i,NAM-44i,EUR-44i,AFR-44i,WAS-44i,EAS-44i,CAS-44i,AUS-44i,ANT-44i,ARC-44i,MED-44i,EUR-11i,MNA-44,MNA-44i,MNA-22,MNA-22i
driving_model_options = ERAINT, ECMWF-ERAINT, CCCma-CanESM2, CNRM-CERFACS-CNRM-CM5, ICHEC-EC-EARTH, MIROC-MIROC5, MOHC-HadGEM2-ES, MPI-M-MPI-ESM-LR, NCC-NorESM1-M, NOAA-GFDL-GFDL-ESM2M, IPSL-IPSL-CM5A-MR
ensemble_options = r1i1p1, r12i1p1, r0i0p0
experiment_options =
        cordex | evaluation | no description
        cordex | historical | no description
        cordex | rcp4 | no description
        cordex | rcp26 | no description
        cordex | rcp45 | no description
        cordex | rcp85 | no description
institute_map = map(project_id,model : institute)
        cordex | WRF331G | UCAN
institute_options = UCAN
las_configure = false
las_time_delta_map = map(time_frequency : las_time_delta)
        mon     | 1 month
        day     | 1 day
        fx      | fixed
        sem     | semi
maps = institute_map, las_time_delta_map, domain_map
model_options = WRF331G
parent_id = wdcc2.cordex
project_handler_name = basic_builtin
rcm_model_options = UCAN-WRF331G
rcm_version_options = v02
thredds_exclude_variables = a, a_bnds, alev1, alevel, alevhalf, alt40, b, b_bnds, basin, bnds, bounds_lat, bounds_lon, dbze, depth, depth0m, depth100m, depth_bnds, geo_region, height, height10m, height2m, Lambert_Conformal, lat, lat_bnds, lat_bounds, latitude, latitude_bnds, layer, lev, lev_bnds, location, lon, lon_bnds, lon_bounds, longitude, longitude_bnds, olayer100m, olevel, oline, p0, p220, p500, p560, p700, p840, plev, plev3, plev7, plev8, plev_bnds, plevs, pressure1, region, rho, rlat, rotated_pole, rlon, scatratio, sdepth, sdepth1, sza5, tau, tau_bnds, time, time1, time2, time_bnds, vegtype, x, y
time_frequency_options = day,fx,mon,sem,3hr,6hr
variable_locate = ps,ps_ | vas,vas_ | uas,uas_ |tasmin,tasmin_|tasmax,tasmax_| tas,tas_ | sfcWindmax,sfcWindmax_ | sfcWind,sfcWind_ |rsds,rsds_|rlds,rlds_| pr,pr_ | prc,prc_| mrso,mrso_| mrros,mrros_|huss,huss_| hfss,hfss_|hfls,hfls_
variable_per_file = true
version_options = 20131108

Therefore, if you use the above configuration file, you will have to create a tree directory like this:

[esgf@data datasets]# tree
.
`-- CORDEX
    `-- output
        `-- EUR-22
            `-- UCAN
                `-- ECMWF-ERAINT
                    `-- evaluation
                        `-- r1i1p1
                            `-- UCAN-WRF311G
                                `-- v02
                                    |-- 3hr
                                    |   |-- hfls
                                    |   |   `-- 20131108
                                    |   |       |-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       `-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   |-- hfss
                                    |   |   `-- 20131108
                                    |   |       |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc
                                    |   |       `-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   |-- huss
                                    |   |   `-- 20131108
                                    |   |       |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc
                                    |   |       `-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   |-- pr
                                    |   |   `-- 20131108
                                    |   |       |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   |-- prc
                                    |   |   `-- 20131108
                                    |   |       |-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       `-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   |-- ps
                                    |   |   `-- 20131108
                                    |   |       |-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       `-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   |-- rlds
                                    |   |   `-- 20131108
                                    |   |       |-- rlds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       |-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc
                                    |   |       `-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   |-- sfcWind
                                    |   |   `-- 20131108
                                    |   |       |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |   |       |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc
                                    |   |       |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc
                                    |   |       `-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |   `-- tas
                                    |       `-- 20131108
                                    |           |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
                                    |           `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
                                    |-- 6hr
                                    |   |-- mrros
                                    |   |   `-- 20131108
                                    |   |       |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc
                                    |   |       |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc
                                    |   |       `-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc
                                    |   |-- uas
                                    |   |   `-- 20131108
                                    |   |       |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc
                                    |   |       |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc
                                    |   |       `-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc
                                    |   `-- vas
                                    |       `-- 20131108
                                    |           |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc
                                    |           |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc
                                    |           `-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc
                                    |-- day
                                    |   |-- mrso
                                    |   |   `-- 20131108
                                    |   |       |-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
                                    |   |       `-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
                                    |   |-- pr
                                    |   |   `-- 20131108
                                    |   |       |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
                                    |   |       `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
                                    |   |-- sfcWindmax
                                    |   |   `-- 20131108
                                    |   |       |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
                                    |   |       |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20010101-20051231.nc
                                    |   |       `-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
                                    |   |-- tasmax
                                    |   |   `-- 20131108
                                    |   |       |-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
                                    |   |       `-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
                                    |   `-- tasmin
                                    |       `-- 20131108
                                    |           |-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
                                    |           `-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
                                    `-- mon
                                        |-- pr
                                        |   `-- 20131108
                                        |       |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc
                                        |       `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc
                                        `-- tas
                                            `-- 20131108
                                                |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc
                                                |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19910101-20001231.nc
                                                `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc


Then you have to add the project name to the esgcet_models_table.txt file

$ echo "   cordex | WRF331G | http://meteo.unican.es | UNICAN WRF3.3.1 Model version, 2.0" >> /esg/config/esgcet/esgcet_models_table.txt 

After modifying esgcet_models_table.txt and esg.ini files, you have to update the data base by executing :

$ cd /usr/local/uvcdat/1.4.0/bin/
$ ./esginitialize -i /esg/config/esgcet/esg.ini -c

Using the ESGF Publisher

This takes place in three steps:

File Scan Phase

In order to scan the cordex files for metadata, run esgscan_directory to generate a mapfile and after that run esgpublish with input from a mapfile:

$ whoami 
esgf
$ cd /usr/local/uvcdat/1.4.0/bin
$ ./esgscan_directory -i /esg/config/esgcet/esg.ini --project cordex -o ~/cordex.txt /datasets/CORDEX
$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt 

In order to check that the datasets were entered correctly into the database, list the datasets for cordex project:

$ ./esglist_datasets -i /esg/config/esgcet/esg.ini cordex
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| id | name                                                                         | project | model   | experiment | run_name | offline | master_gateway | domain | institute | publish_time        | publish_status  | parent | version | version_name                                                                    | comment |
+--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
| 1  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:21 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1       | None    |
| 2  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:25 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1       | None    |
| 3  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:29 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1       | None    |
| 4  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:33 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1         | None    |
| 5  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:37 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1        | None    |
| 6  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:40 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps.v1         | None    |
| 7  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:44 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds.v1       | None    |
| 8  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:48 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds.v1       | None    |
| 9  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind    | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:52 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind.v1    | None    |
| 10 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:56 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas.v1        | None    |
| 11 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros      | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:00 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros.v1      | None    |
| 12 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:06 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas.v1        | None    |
| 13 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:12 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas.v1        | None    |
| 14 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:13 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso.v1       | None    |
| 15 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:14 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr.v1         | None    |
| 16 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:15 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax.v1 | None    |
| 17 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax     | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:16 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax.v1     | None    |
| 18 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin     | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin.v1     | None    |
| 19 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr.v1         | None    |
| 20 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:19 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas.v1        | None    |
+------------------------------------------------------------------------------------------------------------------------------------------------

In order undo the scan, run esgunpublish with the following flags:

$ ./esgunpublish  -i /esg/config/esgcet/esg.ini --skip-gateway --skip-thredds --database-delete --map ~/cordex.txt

Generate a THREDDS catalog

You can generate the THREDDS catalog with :

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds
INFO       2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml
INFO       2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml
INFO       2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml
INFO       2013-11-19 19:46:31,198 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1.xml
INFO       2013-11-19 19:46:31,536 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1.xml
...............................................

In order to remove the catalogs from the THREDDS :

$ ./esgunpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-gateway

idx notification

First, obtain a digital certificate from an ESGF trusted MyProxy server, and rename it to whatever path you have defined in esg.ini.

Remember, you have to log in a Federation to do it.

$ sudo /usr/local/globus/bin/myproxy-logon -s vesgint-idx.ipsl.jussieu.fr -l blancojc -o ~/.globus/certificate-file

Then you can publish the cordex catalog by executing :

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --publish
INFO       2013-11-19 20:01:24,817 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls
INFO       2013-11-19 20:01:28,678   Result: SUCCESSFUL
INFO       2013-11-19 20:01:28,678 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss
INFO       2013-11-19 20:01:32,416   Result: SUCCESSFUL
INFO       2013-11-19 20:01:32,417 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss
INFO       2013-11-19 20:01:36,125   Result: SUCCESSFUL
INFO       2013-11-19 20:01:36,125 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr
INFO       2013-11-19 20:01:39,964   Result: SUCCESSFUL
INFO       2013-11-19 20:01:39,965 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc

Use esgunpublish to delete idx datasets:

$ ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-thredds

Running all publication steps

For convenience, the full publication can be performed with one command. Also, if the arguments are directories rather than a mapfile, the directories will be scanned as if esgscan_directory were run:

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds
INFO       2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml
INFO       2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml
INFO       2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml

esgunpublish will remove the datasets from the idx, THREDDS, and node database in that order:

$ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --database-delete --map ~/cordex.txt

Access files

Finally, in order to grant access to our files you need to add the line below :

<policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/>

in your esgf_policies_local.xml file :

$ cat /usr/local/apache-tomcat-7.0.47/webapps/esg-orp/WEB-INF/classes/esg/orp/orp/config/esgf_policies_local.xml
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>

<!-- This file is intended to be edited and maintained by the local Node admnistrators.
     It should contain only policies specific to data served by the local Node.
     It will not be overridden by a software update. -->
<policies xmlns="http://www.esgf.org/security">
     <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/>
    <!-- The following statements allow all members of group "CMIP5 Research" or "CMIP5 Commercial" to read any local URL that contains "cmip5".
         Note that the groups "CMIP5 Research" and "CMIP5 Commercial" are administered by PCMDI -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="user" action="Read"/> -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="user" action="Read"/> -->
    <!-- These statements provide CMIP5 Read access for members of the old gateways -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="default" action="Read"/> -->
    <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="default" action="Read"/> -->
    

    <!-- The following statement allows all members of group "MY GROUP" to read any local URL that contains "my_data"
    <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="user" action="Read"/> -->
    
    <!-- The following statement allows members of group "MY GROUP" with role="publisher" to publish local datasets with id containing "my_data" 
    <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="publisher" action="Write"/> -->
    
    <!-- The following statements makes resources that contain '.*test.*' freely available for download -->
    <!-- <policy resource=".*test.*" attribute_type="ANY" attribute_value="" action="Read"/> -->
    
    <!-- The following statements makes all resources freely available for download -->
    <!-- <policy resource=".*" attribute_type="ANY" attribute_value="" action="Read"/> -->

</policies>

In production status, you will have to something like this:

<policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="user" action="Read"/>
<policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="default" action="Read"/>
<policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="user" action="Read"/>
<policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="default" action="Read"/>

v20140328