Changes between Version 89 and Version 90 of ESGFNodeInstallation


Ignore:
Timestamp:
Apr 10, 2014 9:19:54 AM (7 years ago)
Author:
antonio
Comment:

--

Legend:

Unmodified
Added
Removed
Modified
  • ESGFNodeInstallation

    v89 v90  
    351351}}}
    352352
    353 = Data Publishing =
    354 
    355 == Configuring a new project for ESGF publication ==
    356 
    357 
    358 [[NoteBox(tip, See the [http://esg-pcmdi.llnl.gov/internal/esg-data-node-documentation/introduction-to-esg-data-node-configuration/ ESGF] publication reference for details.)]]
    359 
    360 
    361 In order to publish, you have to configure a text file, `/esg/config/esgcet/esg.ini`. For this propose, we are going to create a new project called `cordex`:
    362 
    363 
    364 {{{
    365 #!sh
    366 [initialize]
    367 log_level = DEBUG
    368 initial_models_table = /esg/config/esgcet/esgcet_models_table.txt
    369 
    370 
    371 [DEFAULT]
    372 thredds_dataset_roots =
    373         esg_dataroot | /datasets
    374 
    375 project_options =
    376         cordex | CORDEX Output data | 1
    377 [project:cordex]
    378 #       name | category_type | is_mandatory | is_thredds_property | display_order
    379 categories =
    380         project          | enum | true | true | 0
    381         domain           | enum | true | true | 1
    382         institute        | enum | true | true | 2
    383         driving_model    | enum | false | true | 3
    384         experiment       | enum | false | true | 4
    385         ensemble         | enum | false | true | 5
    386         model            | enum | false | true | 6
    387         time_frequency   | enum | false | true | 7
    388         version          | enum | false | true | 8
    389         rcm_model        | enum | false | true | 9
    390         rcm_version      | enum | false | true | 10
    391         description      | text | false | false | 99
    392 category_defaults =
    393         domain | EUR-22
    394         institute | UCAN
    395         driving_model | ERAINT
    396         ensemble | r1i1p1
    397         model | WRF331G
    398         time_frequency| mon
    399 dataset_id = cordex.%(domain)s.%(institute)s.%(driving_model)s.%(experiment)s.%(ensemble)s.WRF331G_v02.%(time_frequency)s.%(variable)s
    400 directory_format = /datasets/CORDEX/output/%(domain)s/%(institute)s/%(driving_model)s/%(experiment)s/%(ensemble)s/%(rcm_model)s/%(rcm_version)s/%(time_frequency)s/%(variable)s/%(version)s
    401 domain_map = map(project_id,domain : domain_description)
    402         cordex | SAM-44 | South America
    403         cordex | CAM-44 | Central America
    404         cordex | NAM-44 | North America
    405         cordex | EUR-44 | Europe
    406         cordex | EUR-22 | Europe
    407         cordex | AFR-44 | Africa
    408         cordex | WAS-44 | West Asia
    409         cordex | EAS-44 | East Asia
    410         cordex | CAS-44 | Central Asia
    411         cordex | AUS-44 | Australasia
    412         cordex | ANT-44 | Antarctica
    413         cordex | ARC-44 | The Arctic
    414         cordex | MED-44 | HYMEX Mediterranean
    415         cordex | EUR-11 | High-res. Europe
    416         cordex | SAM-44i | South America
    417         cordex | CAM-44i | Central America
    418         cordex | NAM-44i | North America
    419         cordex | EUR-44i | Europe
    420         cordex | AFR-44i | Africa
    421         cordex | WAS-44i | West Asia
    422         cordex | EAS-44i | East Asia
    423         cordex | CAS-44i | Central Asia
    424         cordex | AUS-44i | Australasia
    425         cordex | ANT-44i | Antarctica
    426         cordex | ARC-44i | The Arctic
    427         cordex | MED-44i | HYMEX Mediterranean
    428         cordex | EUR-11i | High-res. Europe
    429         cordex | MNA-44  | Middle East and North Africa
    430         cordex | MNA-44i | Middle East and North Africa
    431         cordex | MNA-22  | Middle East and North Africa
    432         cordex | MNA-22i | Middle East and North Africa
    433 domain_options = SAM-44,CAM-44,NAM-44,EUR-44,EUR-22,EUR-44i,AFR-44,AFR-44i,WAS-44,EAS-44,CAS-44,AUS-44,ANT-44,ARC-44,MED-44,EUR-11,SAM-44i,CAM-44i,NAM-44i,EUR-44i,AFR-44i,WAS-44i,EAS-44i,CAS-44i,AUS-44i,ANT-44i,ARC-44i,MED-44i,EUR-11i,MNA-44,MNA-44i,MNA-22,MNA-22i
    434 driving_model_options = ERAINT, ECMWF-ERAINT, CCCma-CanESM2, CNRM-CERFACS-CNRM-CM5, ICHEC-EC-EARTH, MIROC-MIROC5, MOHC-HadGEM2-ES, MPI-M-MPI-ESM-LR, NCC-NorESM1-M, NOAA-GFDL-GFDL-ESM2M, IPSL-IPSL-CM5A-MR
    435 ensemble_options = r1i1p1, r12i1p1, r0i0p0
    436 experiment_options =
    437         cordex | evaluation | no description
    438         cordex | historical | no description
    439         cordex | rcp4 | no description
    440         cordex | rcp26 | no description
    441         cordex | rcp45 | no description
    442         cordex | rcp85 | no description
    443 institute_map = map(project_id,model : institute)
    444         cordex | WRF331G | UCAN
    445 institute_options = UCAN
    446 las_configure = false
    447 las_time_delta_map = map(time_frequency : las_time_delta)
    448         mon     | 1 month
    449         day     | 1 day
    450         fx      | fixed
    451         sem     | semi
    452 maps = institute_map, las_time_delta_map, domain_map
    453 model_options = WRF331G
    454 parent_id = wdcc2.cordex
    455 project_handler_name = basic_builtin
    456 rcm_model_options = UCAN-WRF331G
    457 rcm_version_options = v02
    458 thredds_exclude_variables = a, a_bnds, alev1, alevel, alevhalf, alt40, b, b_bnds, basin, bnds, bounds_lat, bounds_lon, dbze, depth, depth0m, depth100m, depth_bnds, geo_region, height, height10m, height2m, Lambert_Conformal, lat, lat_bnds, lat_bounds, latitude, latitude_bnds, layer, lev, lev_bnds, location, lon, lon_bnds, lon_bounds, longitude, longitude_bnds, olayer100m, olevel, oline, p0, p220, p500, p560, p700, p840, plev, plev3, plev7, plev8, plev_bnds, plevs, pressure1, region, rho, rlat, rotated_pole, rlon, scatratio, sdepth, sdepth1, sza5, tau, tau_bnds, time, time1, time2, time_bnds, vegtype, x, y
    459 time_frequency_options = day,fx,mon,sem,3hr,6hr
    460 variable_locate = ps,ps_ | vas,vas_ | uas,uas_ |tasmin,tasmin_|tasmax,tasmax_| tas,tas_ | sfcWindmax,sfcWindmax_ | sfcWind,sfcWind_ |rsds,rsds_|rlds,rlds_| pr,pr_ | prc,prc_| mrso,mrso_| mrros,mrros_|huss,huss_| hfss,hfss_|hfls,hfls_
    461 variable_per_file = true
    462 version_options = 20131108
    463 }}}
    464 
    465 Therefore, if you use the above configuration file, you will have to create a tree directory like this:
    466 
    467 {{{
    468 
    469 [esgf@data datasets]# tree
    470 .
    471 `-- CORDEX
    472     `-- output
    473         `-- EUR-22
    474             `-- UCAN
    475                 `-- ECMWF-ERAINT
    476                     `-- evaluation
    477                         `-- r1i1p1
    478                             `-- UCAN-WRF311G
    479                                 `-- v02
    480                                     |-- 3hr
    481                                     |   |-- hfls
    482                                     |   |   `-- 20131108
    483                                     |   |       |-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    484                                     |   |       `-- hfls_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    485                                     |   |-- hfss
    486                                     |   |   `-- 20131108
    487                                     |   |       |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    488                                     |   |       |-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc
    489                                     |   |       `-- hfss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    490                                     |   |-- huss
    491                                     |   |   `-- 20131108
    492                                     |   |       |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    493                                     |   |       |-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc
    494                                     |   |       `-- huss_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    495                                     |   |-- pr
    496                                     |   |   `-- 20131108
    497                                     |   |       |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    498                                     |   |       `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    499                                     |   |-- prc
    500                                     |   |   `-- 20131108
    501                                     |   |       |-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    502                                     |   |       `-- prc_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    503                                     |   |-- ps
    504                                     |   |   `-- 20131108
    505                                     |   |       |-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    506                                     |   |       `-- ps_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    507                                     |   |-- rlds
    508                                     |   |   `-- 20131108
    509                                     |   |       |-- rlds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    510                                     |   |       |-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc
    511                                     |   |       `-- rsds_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    512                                     |   |-- sfcWind
    513                                     |   |   `-- 20131108
    514                                     |   |       |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    515                                     |   |       |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19800101-19801231.nc
    516                                     |   |       |-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20090101-20091231.nc
    517                                     |   |       `-- sfcWind_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    518                                     |   `-- tas
    519                                     |       `-- 20131108
    520                                     |           |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_19790101-19791231.nc
    521                                     |           `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_3hr_20100101-20101231.nc
    522                                     |-- 6hr
    523                                     |   |-- mrros
    524                                     |   |   `-- 20131108
    525                                     |   |       |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc
    526                                     |   |       |-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc
    527                                     |   |       `-- mrros_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc
    528                                     |   |-- uas
    529                                     |   |   `-- 20131108
    530                                     |   |       |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc
    531                                     |   |       |-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc
    532                                     |   |       `-- uas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc
    533                                     |   `-- vas
    534                                     |       `-- 20131108
    535                                     |           |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_19790101-19791231.nc
    536                                     |           |-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20090101-20091231.nc
    537                                     |           `-- vas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_6hr_20100101-20101231.nc
    538                                     |-- day
    539                                     |   |-- mrso
    540                                     |   |   `-- 20131108
    541                                     |   |       |-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
    542                                     |   |       `-- mrso_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
    543                                     |   |-- pr
    544                                     |   |   `-- 20131108
    545                                     |   |       |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
    546                                     |   |       `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
    547                                     |   |-- sfcWindmax
    548                                     |   |   `-- 20131108
    549                                     |   |       |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
    550                                     |   |       |-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20010101-20051231.nc
    551                                     |   |       `-- sfcWindmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
    552                                     |   |-- tasmax
    553                                     |   |   `-- 20131108
    554                                     |   |       |-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
    555                                     |   |       `-- tasmax_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
    556                                     |   `-- tasmin
    557                                     |       `-- 20131108
    558                                     |           |-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_19790101-19801231.nc
    559                                     |           `-- tasmin_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_day_20060101-20101231.nc
    560                                     `-- mon
    561                                         |-- pr
    562                                         |   `-- 20131108
    563                                         |       |-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc
    564                                         |       `-- pr_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc
    565                                         `-- tas
    566                                             `-- 20131108
    567                                                 |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19790101-19801231.nc
    568                                                 |-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_19910101-20001231.nc
    569                                                 `-- tas_EUR-22_ECMWF-ERAINT_evaluation_r1i1p1_UCAN-WRF331G_v02_mon_20010101-20101231.nc
    570 
    571 
    572 }}}
    573 
    574 
    575 Then you have to add the project name to the `esgcet_models_table.txt` file
    576 
    577 {{{
    578 #!sh
    579 $ echo "   cordex | WRF331G | http://meteo.unican.es | UNICAN WRF3.3.1 Model version, 2.0" >> /esg/config/esgcet/esgcet_models_table.txt
    580 }}}
    581 
    582 
    583 After modifying `esgcet_models_table.txt` and `esg.ini` files, you have to update the data base by executing :
    584 {{{
    585 #!sh
    586 $ cd /usr/local/uvcdat/1.4.0/bin/
    587 $ ./esginitialize -i /esg/config/esgcet/esg.ini -c
    588 }}}
    589 
    590 * `./esginitialize -d 0`: To remove all tables
    591 
    592 * `./esginitialize -c`  : Upgrade the database schema to the latest version, and initialize projects, models, experiments, and standard names from configuration files.
    593 
    594 == Using the ESGF Publisher ==
    595 
    596 
    597 This takes place in three steps:
    598 
    599 * Scan each file for metadata and save the metadata in the node database. (This is in contrast to running `esgscan_directory`, which just scans the directory structure.)
    600 
    601 * Generate a THREDDS catalog based on the scanned information. THREDDS is a data and metadata server used by ESGF.
    602 
    603 * Notify the idx that one or more catalogs have been generated.
    604 
    605 === File Scan Phase ===
    606 
    607 In order to scan the cordex files for metadata, run `esgscan_directory` to generate a mapfile and after that run `esgpublish` with input from a mapfile:
    608 {{{
    609 #!sh
    610 $ whoami
    611 esgf
    612 $ cd /usr/local/uvcdat/1.4.0/bin
    613 $ ./esgscan_directory -i /esg/config/esgcet/esg.ini --project cordex -o ~/cordex.txt /datasets/CORDEX
    614 $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt
    615 }}}
    616 
    617 In order to check that the datasets were entered correctly into the database, list the datasets for cordex project:
    618 
    619 {{{
    620 #!sh
    621 $ ./esglist_datasets -i /esg/config/esgcet/esg.ini cordex
    622 +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
    623 | id | name                                                                         | project | model   | experiment | run_name | offline | master_gateway | domain | institute | publish_time        | publish_status  | parent | version | version_name                                                                    | comment |
    624 +--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+
    625 | 1  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:21 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1       | None    |
    626 | 2  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:25 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1       | None    |
    627 | 3  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:29 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1       | None    |
    628 | 4  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:33 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1         | None    |
    629 | 5  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:37 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1        | None    |
    630 | 6  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:40 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.ps.v1         | None    |
    631 | 7  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:44 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rlds.v1       | None    |
    632 | 8  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:48 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.rsds.v1       | None    |
    633 | 9  | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind    | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:52 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.sfcWind.v1    | None    |
    634 | 10 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:42:56 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.tas.v1        | None    |
    635 | 11 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros      | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:00 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.mrros.v1      | None    |
    636 | 12 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:06 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.uas.v1        | None    |
    637 | 13 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:12 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.6hr.vas.v1        | None    |
    638 | 14 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso       | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:13 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.mrso.v1       | None    |
    639 | 15 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:14 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.pr.v1         | None    |
    640 | 16 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:15 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.sfcWindmax.v1 | None    |
    641 | 17 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax     | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:16 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmax.v1     | None    |
    642 | 18 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin     | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.day.tasmin.v1     | None    |
    643 | 19 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr         | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:18 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.pr.v1         | None    |
    644 | 20 | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas        | cordex  | WRF331G | evaluation | None     | False   | None           | EUR-22 | UCAN      | 2013-11-18 19:43:19 | PUBLISH_DATASET | ROOT   | 1       | cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.mon.tas.v1        | None    |
    645 +------------------------------------------------------------------------------------------------------------------------------------------------
    646 
    647 }}}
    648 
    649 In order undo the scan, run `esgunpublish` with the following flags:
    650 
    651 {{{
    652 #!sh
    653 $ ./esgunpublish  -i /esg/config/esgcet/esg.ini --skip-gateway --skip-thredds --database-delete --map ~/cordex.txt
    654 }}}
    655 
    656 ===  Generate a THREDDS catalog ===
    657 
    658 You can generate the THREDDS catalog with :
    659 
    660 {{{
    661 #!sh
    662 $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds
    663 INFO       2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml
    664 INFO       2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml
    665 INFO       2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml
    666 INFO       2013-11-19 19:46:31,198 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr.v1.xml
    667 INFO       2013-11-19 19:46:31,536 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc.v1.xml
    668 ...............................................
    669 }}}
    670 
    671 
    672 In order to remove the catalogs from the THREDDS :
    673 
    674 {{{
    675 #!sh
    676 $ ./esgunpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-gateway
    677 }}}
    678 
    679 === idx notification ===
    680 
    681 First, obtain a digital certificate from an ESGF trusted !MyProxy server, and rename it to whatever path you have defined in esg.ini.
    682 
    683 [[NoteBox(warn, Remember\, you have to log in a Federation to do it.)]]
    684 
    685 {{{
    686 #!sh
    687 $ sudo /usr/local/globus/bin/myproxy-logon -s vesgint-idx.ipsl.jussieu.fr -l blancojc -o ~/.globus/certificate-file
    688 }}}
    689 
    690 Then you can publish the cordex catalog by executing :
    691 
    692 {{{
    693 #!sh
    694 $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --publish
    695 INFO       2013-11-19 20:01:24,817 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls
    696 INFO       2013-11-19 20:01:28,678   Result: SUCCESSFUL
    697 INFO       2013-11-19 20:01:28,678 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss
    698 INFO       2013-11-19 20:01:32,416   Result: SUCCESSFUL
    699 INFO       2013-11-19 20:01:32,417 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss
    700 INFO       2013-11-19 20:01:36,125   Result: SUCCESSFUL
    701 INFO       2013-11-19 20:01:36,125 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.pr
    702 INFO       2013-11-19 20:01:39,964   Result: SUCCESSFUL
    703 INFO       2013-11-19 20:01:39,965 Publishing: cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.prc
    704 }}}
    705 
    706 Use `esgunpublish` to delete idx datasets:
    707 
    708 {{{
    709 #!sh
    710 $ ./esgpublish -i /esg/config/esgcet/esg.ini --map ~/cordex.txt --skip-thredds
    711 }}}
    712 
    713 
    714 === Running all publication steps ===
    715 
    716 For convenience, the full publication can be performed with one command. Also, if the arguments are directories rather than a mapfile, the directories will be scanned as if esgscan_directory were run:
    717 
    718 {{{
    719 #!sh
    720 $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --project cordex --map ~/cordex.txt --noscan --thredds
    721 INFO       2013-11-19 19:46:30,642 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfls.v1.xml
    722 INFO       2013-11-19 19:46:30,837 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.hfss.v1.xml
    723 INFO       2013-11-19 19:46:31,019 Writing THREDDS catalog /esg/content/thredds/esgcet/1/cordex.EUR-22.UCAN.ECMWF-ERAINT.evaluation.r1i1p1.WRF331G_v02.3hr.huss.v1.xml
    724 }}}
    725 
    726 `esgunpublish` will remove the datasets from the idx, THREDDS, and node database in that order:
    727 
    728 {{{
    729 #!sh
    730 $ sudo ./esgpublish -i /esg/config/esgcet/esg.ini --database-delete --map ~/cordex.txt
    731 }}}
    732 
    733 === Access files ===
    734 
    735 Finally, in order to grant access to our files you need to add the line below :
    736 
    737 {{{
    738 <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/>
    739 }}}
    740 
    741 in your `esgf_policies_local.xml` file :
    742 
    743 {{{
    744 #!sh
    745 $ cat /usr/local/apache-tomcat-7.0.47/webapps/esg-orp/WEB-INF/classes/esg/orp/orp/config/esgf_policies_local.xml
    746 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
    747 
    748 <!-- This file is intended to be edited and maintained by the local Node admnistrators.
    749      It should contain only policies specific to data served by the local Node.
    750      It will not be overridden by a software update. -->
    751 <policies xmlns="http://www.esgf.org/security">
    752      <policy resource=".*CORDEX.*" attribute_type="ANY" attribute_value="" action="Read"/>
    753     <!-- The following statements allow all members of group "CMIP5 Research" or "CMIP5 Commercial" to read any local URL that contains "cmip5".
    754          Note that the groups "CMIP5 Research" and "CMIP5 Commercial" are administered by PCMDI -->
    755     <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="user" action="Read"/> -->
    756     <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="user" action="Read"/> -->
    757     <!-- These statements provide CMIP5 Read access for members of the old gateways -->
    758     <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Research" attribute_value="default" action="Read"/> -->
    759     <!-- <policy resource=".*cmip5.*" attribute_type="CMIP5 Commercial" attribute_value="default" action="Read"/> -->
    760    
    761 
    762     <!-- The following statement allows all members of group "MY GROUP" to read any local URL that contains "my_data"
    763     <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="user" action="Read"/> -->
    764    
    765     <!-- The following statement allows members of group "MY GROUP" with role="publisher" to publish local datasets with id containing "my_data"
    766     <policy resource=".*my_data.*" attribute_type="MY GROUP" attribute_value="publisher" action="Write"/> -->
    767    
    768     <!-- The following statements makes resources that contain '.*test.*' freely available for download -->
    769     <!-- <policy resource=".*test.*" attribute_type="ANY" attribute_value="" action="Read"/> -->
    770    
    771     <!-- The following statements makes all resources freely available for download -->
    772     <!-- <policy resource=".*" attribute_type="ANY" attribute_value="" action="Read"/> -->
    773 
    774 </policies>
    775 }}}
    776 
    777 
    778 
    779 In production status, you will have to something like this:
    780 {{{
    781 <policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="user" action="Read"/>
    782 <policy resource=".*CORDEX.*" attribute_type="CORDEX_Research" attribute_value="default" action="Read"/>
    783 <policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="user" action="Read"/>
    784 <policy resource=".*CORDEX.*" attribute_type="CORDEX_Commercial" attribute_value="default" action="Read"/>
    785 }}}
    786 
    787