Litef Conductor

The Litef Conductor can be built and installed from its source in the litef-conductor repository on GitHub.

To build it, Java 1.7.x, Scala 2.10.4 and SBT are required. Running

sbt compile assembly

will compile a fat jar, that can be deployed to the server.

Additionally, Litef requires a configuration file /etc/litef/application.conf

logger.scala.slick.session=DEBUG

litef {
  namespace = "http://localhost:42042/v1/"
}

litef.ckan {
  home         = "https://host/ckan"
  namespace    = "https://host/ckan/api/3/"
  apiKey       = "***********"

  localStoragePrefix = "/data/resources"

  postgres {
    url      = "jdbc:postgresql://localhost:5432/***********"
    driver   = "org.postgresql.Driver"
    username = "***********"
    password = "***********"
  }
}

litef.virtuoso {
  url = "jdbc:virtuoso://localhost:1111"
  username = "dba"
  password = "***********"
}

litef.nerd {
  namespace = "http://traces1.saclay.inria.fr:8090/"
}

litef.elastic {
  namespace = "http://localhost:9200/"
}

litef.indexer {
  localStoragePrefix = "/var/lib/litef/default/resources"
}

litef.conductor {
  fileSizeLimit = 10000
  plugins = "conductor.plugins.DocumentIndexerPlugin,conductor.plugins.NerdPlugin,conductor.plugins.VirtuosoFeederPlugin,conductor.plugins.ElasticFeederPlugin"
}

akka {
  log-dead-letters = 10
  log-dead-letters-during-shutdown = on
  loglevel = "INFO"
}

spray {
  can.server {
    request-timeout = 240s
    idle-timeout = 300s
  }
  can.client {
    request-timeout = 240s
    idle-timeout = 300s
  }

  routing {
    file-chunking-threshold-size = 10k
    file-chunking-chunk-size = 5k
  }
}

logger.scala.slick=ERROR
logger.scala.slick.session=ERROR

Litef requires a dedicated CKAN sysadmin, thus after installing CKAN create that sysadmin:

source /var/www/ckan/bin/activate
cd /var/www/ckan/src/ckan
paster sysadmin add litef -c /etc/ckan/production.ini

Add the new sysadmin’s API key to /etc/litef/application.conf.

Litef needs modifications to the CKAN database.

CREATE VIEW litef_ckan_resource_view AS
SELECT resource.id,
 resource.resource_group_id,
 resource.url,
 resource.format,
 resource.description,
 resource."position",
 resource.revision_id,
 resource.hash,
 resource.state,
 resource.extras,
 resource.name,
 resource.resource_type,
 resource.mimetype,
 resource.mimetype_inner,
 resource.size,
 resource.last_modified,
 resource.cache_url,
 resource.cache_last_updated,
 resource.webstore_url,
 resource.webstore_last_updated,
 resource.created,
 GREATEST(resource.last_modified, resource.created) AS modified,
 ( SELECT rg.package_id
   FROM resource_group rg
   WHERE (rg.id = resource.resource_group_id)) AS package_id,
 resource.url_type
FROM resource;

ALTER TABLE public.litef_ckan_resource_view OWNER TO ckanuser;

CREATE VIEW litef_ckan_group_resource_view AS
SELECT p.owner_org AS group_id,
 r.id,
 r.resource_group_id,
 r.url,
 r.format,
 r.description,
 r."position",
 r.revision_id,
 r.hash,
 r.state,
 r.extras,
 r.name,
 r.resource_type,
 r.mimetype,
 r.mimetype_inner,
 r.size,
 r.last_modified,
 r.cache_url,
 r.cache_last_updated,
 r.webstore_url,
 r.webstore_last_updated,
 r.created,
 r.modified,
 p.id AS package_id,
 r.url_type
FROM ((package p
  JOIN resource_group rg ON ((p.id = rg.package_id)))
  JOIN litef_ckan_resource_view r ON ((r.resource_group_id = rg.id)));

ALTER TABLE public.litef_ckan_group_resource_view OWNER TO ckanuser;

CREATE VIEW litef_ckan_user_group_role_view AS
SELECT "user".id AS user_id,
 "user".apikey AS user_apikey,
 member.capacity AS group_role,
 member.group_id,
 member.id,
 member.state
FROM (member
  JOIN "user" ON ((member.table_id = "user".id)))
WHERE (member.table_name = 'user'::text);

ALTER TABLE public.litef_ckan_user_group_role_view OWNER TO ckanuser;

CREATE VIEW litef_ckan_user_group_role_view__through_package_group AS
SELECT "user".id AS user_id,
 "user".apikey AS user_apikey,
 object_role.role AS group_role,
 role.group_id
FROM (("user" "user"
  JOIN user_object_role object_role ON (("user".id = object_role.user_id)))
  JOIN group_role role ON ((object_role.id = role.user_object_role_id)));

ALTER TABLE public.litef_ckan_user_group_role_view__through_package_group OWNER TO ckanuser;

CREATE VIEW litef_ckan_user_package_role_view AS
SELECT "user".id AS user_id,
 "user".apikey AS user_apikey,
 object_role.role AS package_role,
 role.package_id
FROM (("user" "user"
  JOIN user_object_role object_role ON (("user".id = object_role.user_id)))
  JOIN package_role role ON ((object_role.id = role.user_object_role_id)));

ALTER TABLE public.litef_ckan_user_package_role_view OWNER TO ckanuser;

CREATE VIEW litef_ckan_user_package_view AS
SELECT group_r.user_id,
 group_r.user_apikey,
 group_r.group_role AS package_role,
 package.id,
 package.name,
 package.title,
 package.version,
 package.url,
 package.notes,
 package.license_id,
 package.revision_id,
 package.author,
 package.author_email,
 package.maintainer,
 package.maintainer_email,
 package.state,
 package.type,
 package.owner_org,
 package.private
FROM ((litef_ckan_user_group_role_view__through_package_group group_r
  JOIN "group" ON ((group_r.group_id = "group".id)))
  JOIN package ON ((package.owner_org = "group".id)));

ALTER TABLE public.litef_ckan_user_package_view OWNER TO ckanuser;

CREATE TABLE litef_processed_resource (
 resource_id character varying(254) NOT NULL,
 last_processed timestamp without time zone,
 attachment character varying NOT NULL);

ALTER TABLE public.litef_processed_resource OWNER TO ckanuser;

CREATE TABLE litef_resource_attachment (
 resource_id character varying(254) NOT NULL,
 format character varying(254) NOT NULL,
 created timestamp without time zone NOT NULL,
 modified timestamp without time zone NOT NULL,
 content text);

ALTER TABLE public.litef_resource_attachment OWNER TO ckanuser;

ALTER TABLE ONLY litef_processed_resource
 ADD CONSTRAINT litef_processed_resource_pkey PRIMARY KEY (resource_id, attachment);

ALTER TABLE ONLY litef_resource_attachment
 ADD CONSTRAINT litef_resource_attachment_pk PRIMARY KEY (resource_id, format);

CREATE INDEX litef_processed_resource_id_last_processed_idx ON litef_processed_resource USING btree (resource_id, last_processed);

CREATE INDEX litef_processed_resource_last_processed_idx ON litef_processed_resource USING btree (last_processed);

CREATE INDEX litef_resource_attachment_resource_id_modified_idx ON litef_resource_attachment USING btree (resource_id, modified);

Finally, Litef can be started by

java -Dfile.encoding=UTF-8 -cp /etc/litef:/opt/litef/litef-conductor.jar core.Rest

Data API

The Data API is served by Litef on port 42042 and can be accessed locally. To provide remote access from the Front Office server, create a dedicated user named apiuser. When adding the public keys for connecting to its .ssh/authorized_keys, it is recommended to prepend options from="1.2.3.4",no-pty,no-X11-forwarding for security.