~ can generate a stand-alone jar shipping jetty and a SPARQL2SQL endpoint
authorAlexandre Bertails <bertails@w3.org>
Wed, 03 Nov 2010 14:03:52 -0400
changeset 259 c0d7f7cc8c54
parent 250 a5f27d9d047a
child 260 cf60d13d95da
~ can generate a stand-alone jar shipping jetty and a SPARQL2SQL endpoint
.hgignore
project/build/RDB2RDF.scala
project/plugins/Plugins.scala
project/plugins/project/build.properties
sparql2sqlendpoint/src/main/resources/database.properties
sparql2sqlendpoint/src/main/resources/ddl.sql
sparql2sqlendpoint/src/main/resources/ddl.txt
sparql2sqlendpoint/src/main/resources/default-config.json
sparql2sqlendpoint/src/main/resources/default-sparql-query.txt
sparql2sqlendpoint/src/main/resources/rdb2rdf.properties
sparql2sqlendpoint/src/main/scala/Config.scala
sparql2sqlendpoint/src/main/scala/Configuration.scala
sparql2sqlendpoint/src/main/scala/Servlet.scala
sparql2sqlendpoint/src/main/webapp/WEB-INF/web.xml
--- a/.hgignore	Sun Oct 31 20:43:14 2010 -0400
+++ b/.hgignore	Wed Nov 03 14:03:52 2010 -0400
@@ -6,6 +6,7 @@
 lib_managed
 lift_example
 project/boot/
+project/plugins/src_managed
 *~
 *.class
 *.log
--- a/project/build/RDB2RDF.scala	Sun Oct 31 20:43:14 2010 -0400
+++ b/project/build/RDB2RDF.scala	Wed Nov 03 14:03:52 2010 -0400
@@ -8,6 +8,9 @@
 }
 
 class FeDeRate(info: ProjectInfo) extends ParentProject(info) {
+  self =>
+
+  val feDeRatePath = Path.fromFile(".")
 
   lazy val rdb = project("rdb", "rdb", new RDB(_))
   lazy val sql = project("sql", "sql", new SQL(_), rdb)
@@ -16,25 +19,61 @@
   lazy val sparql = project("sparql", "sparql", new SPARQL(_), rdf)
   lazy val sparql2sql = project("sparql2sql", "sparql2sql", new SPARQL2SQL(_), sparql, sql)
   lazy val sparql2sparql = project("sparql2sparql", "sparql2sparql", new SPARQL2SPARQL(_), sparql)
+  lazy val sparql2sparql2sql = project("sparql2sparql2sql", "sparql2sparql2sql", new SPARQL2SPARQL2SQL(_), sparql2sparql, sparql2sql)
   lazy val sparql2sqlendpoint = project("sparql2sqlendpoint", "sparql2sqlendpoint", new SPARQL2SQLEndPoint(_), sparql2sql)
-  lazy val sparql2sparql2sql = project("sparql2sparql2sql", "sparql2sparql2sql", new SPARQL2SPARQL2SQL(_), sparql2sparql, sparql2sql)
+  lazy val sparql2sqlendpointJar = project("sparql2sqlendpointjar", "sparql2sqlendpointjar", new SPARQL2SQLEndPointJar(_), sparql2sqlendpoint)
 
   class RDB(info: ProjectInfo) extends DefaultProject(info) with Common
+
   class SQL(info: ProjectInfo) extends DefaultProject(info) with Common
+
   class RDF(info: ProjectInfo) extends DefaultProject(info) with Common
+
   class DirectMapping(info: ProjectInfo) extends DefaultProject(info) with Common
+
   class SPARQL(info: ProjectInfo) extends DefaultProject(info) with Common
+
   class SPARQL2SQL(info: ProjectInfo) extends DefaultProject(info) with Common
+
   class SPARQL2SPARQL(info: ProjectInfo) extends DefaultProject(info) with Common
+
   class SPARQL2SQLEndPoint(info: ProjectInfo) extends DefaultWebProject(info) with Common with BSBMPlugin {
-    val jetty6 = "org.mortbay.jetty" % "jetty" % "6.1.14" % "test"
-    val servlet = "javax.servlet" % "servlet-api" % "2.5" % "provided" 
-    val mysql = "mysql" % "mysql-connector-java" % "5.1.12"
+    val jettyConf = config("jetty")
+    /* http://repo1.maven.org/maven2 */
+    val jettyDep = "org.eclipse.jetty" % "jetty-webapp" % "7.0.2.v20100331" % "compile,jetty"
+    override def jettyClasspath = managedClasspath(jettyConf)
+    val mysql = "mysql" % "mysql-connector-java" % "5.1.13"
+    val postgresql = "postgresql" % "postgresql" % "9.0-801.jdbc4"
+    val codaRepo = "Coda Hale's Repository" at "http://repo.codahale.com/"
+    val fig = "com.codahale" %% "fig" % "1.0.5" withSources()
     override def webappUnmanaged = super.webappUnmanaged +++ ("src" / "main" / "resources" / "database.properties")
   }
+
   class SPARQL2SPARQL2SQL(info: ProjectInfo) extends DefaultProject(info) with Common
 
+  class SPARQL2SQLEndPointJar(info: ProjectInfo) extends DefaultProject(info) with ProguardProject {
 
+    // defines the default main class
+    override def mainClass: Option[String] = Some("org.w3.sw.sparql2sqlendpoint.SPARQL2SQLEndpoint")
 
+    // tells Proguard to take some more JAR/WAR/.class
+    override def proguardInJars =
+      super.proguardInJars +++
+      // we need the Scala library
+      scalaLibraryPath +++
+      // the Proguard plugin does not considerer the .war files so we include the generated bytecode
+      (self.feDeRatePath / "sparql2sqlendpoint" / "target" / "scala_2.8.0" / "classes") +++
+      // ships the default configuration
+      (self.feDeRatePath / "sparql2sqlendpoint" / "src" / "main" / "resources")
+
+    override def proguardOptions = List(
+      proguardKeepAllScala,
+      // automatically generates the MANIFEST.MF
+      proguardKeepMain(mainClass.get),
+      // keeps all the SQL drivers we can find
+      "-keep public class * implements java.sql.Driver"
+    )
+
+  }
 
 }
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/project/plugins/Plugins.scala	Wed Nov 03 14:03:52 2010 -0400
@@ -0,0 +1,5 @@
+import sbt._
+
+class Plugins(info: ProjectInfo) extends PluginDefinition(info) {
+  val proguard = "org.scala-tools.sbt" % "sbt-proguard-plugin" % "0.0.5"
+}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/project/plugins/project/build.properties	Wed Nov 03 14:03:52 2010 -0400
@@ -0,0 +1,3 @@
+#Project properties
+#Wed Nov 03 10:16:34 EDT 2010
+plugin.uptodate=true
--- a/sparql2sqlendpoint/src/main/resources/database.properties	Sun Oct 31 20:43:14 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,3 +0,0 @@
-host = localhost
-user = root
-password = 
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sparql2sqlendpoint/src/main/resources/ddl.sql	Wed Nov 03 14:03:52 2010 -0400
@@ -0,0 +1,3 @@
+CREATE TABLE Employee (empid INT, PRIMARY KEY (empid), lastName STRING, birthday DATE, manager INT, FOREIGN KEY (manager) REFERENCES Employee(empid));
+CREATE TABLE Tasks (taskid INT, PRIMARY KEY (taskid), name STRING, lead INT, FOREIGN KEY (lead) REFERENCES Employee(empid));
+CREATE TABLE TaskAssignments (id INT PRIMARY KEY, PRIMARY KEY (id), task INT, FOREIGN KEY (task) REFERENCES Tasks(taskid), employee INT, FOREIGN KEY (employee) REFERENCES Employee(empid));
--- a/sparql2sqlendpoint/src/main/resources/ddl.txt	Sun Oct 31 20:43:14 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,3 +0,0 @@
-CREATE TABLE Employee (empid INT, PRIMARY KEY (empid), lastName STRING, birthday DATE, manager INT, FOREIGN KEY (manager) REFERENCES Employee(empid));
-CREATE TABLE Tasks (taskid INT, PRIMARY KEY (taskid), name STRING, lead INT, FOREIGN KEY (lead) REFERENCES Employee(empid));
-CREATE TABLE TaskAssignments (id INT PRIMARY KEY, PRIMARY KEY (id), task INT, FOREIGN KEY (task) REFERENCES Tasks(taskid), employee INT, FOREIGN KEY (employee) REFERENCES Employee(empid));
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sparql2sqlendpoint/src/main/resources/default-config.json	Wed Nov 03 14:03:52 2010 -0400
@@ -0,0 +1,49 @@
+{
+  // Fig will strip C-style line comments from the JSON.
+  "db": {
+    "driver": "com.mysql.jdbc.Driver",
+    // "driver": "org.postgresql.Driver",
+    "url": "jdbc:mysql://localhost/rdb2rdf",
+    "user": "root",
+    "password": ""
+  },
+  "ui": {
+    "stemuri": "http://foo.example/DB/",
+    "sparql-query": "
+PREFIX asgn: <http://foo.example/DB/TaskAssignments#>
+PREFIX proj: <http://foo.example/DB/Projects#>
+SELECT ?lead
+ WHERE {
+    ?assignment asgn:worker <http://foo.example/DB/People/ID.7#_> .
+    ?assignment asgn:project_deptName_deptCity ?project .
+    ?project proj:lead ?lead .
+ }
+"
+  },
+  // the ddl used for the direct mapping (default one)
+  "default": {
+    "ddl": "
+CREATE TABLE Addresses (ID INT PRIMARY KEY, city STRING, state STRING);
+CREATE TABLE Department (ID INT PRIMARY KEY, name STRING, city STRING, manager INT,
+                         FOREIGN KEY (manager) REFERENCES People(ID),
+                         UNIQUE (name, city));
+CREATE TABLE People (ID INT PRIMARY KEY, fname STRING, addr INT,
+                     FOREIGN KEY (addr) REFERENCES Addresses(ID),
+                     deptName STRING, deptCity STRING,
+                     FOREIGN KEY (deptName, deptCity) REFERENCES Department(name, city));
+CREATE TABLE Projects (lead INT,
+                       FOREIGN KEY (lead) REFERENCES People(ID),
+                       name STRING, UNIQUE (lead, name), 
+                       deptName STRING, deptCity STRING,
+                       UNIQUE (name, deptName, deptCity),
+                       FOREIGN KEY (deptName, deptCity) REFERENCES Department(name, city));
+CREATE TABLE TaskAssignments (worker INT,
+                              FOREIGN KEY (worker) REFERENCES People(ID),
+                              project STRING, PRIMARY KEY (worker, project), 
+                              deptName STRING, deptCity STRING,
+                              FOREIGN KEY (worker) REFERENCES People(ID),
+                              FOREIGN KEY (project, deptName, deptCity) REFERENCES Projects(name, deptName, deptCity),
+                              FOREIGN KEY (deptName, deptCity) REFERENCES Department(name, city));
+"
+  }
+}
--- a/sparql2sqlendpoint/src/main/resources/default-sparql-query.txt	Sun Oct 31 20:43:14 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,17 +0,0 @@
-PREFIX empP : <http://hr.example/DB/Employee#>
-PREFIX task : <http://hr.example/DB/Tasks#>
-PREFIX tass : <http://hr.example/DB/TaskAssignments#>
-PREFIX xsd : <http://www.w3.org/2001/XMLSchema#>
-SELECT ?name ?bday
- WHERE { { ?above   tass:employee  ?who .
-           ?above   tass:task      ?atask .
-           ?atask   task:lead      ?taskLead .
-           ?taskLead empP:lastName  ?name }
-         UNION
-         { ?below   tass:task     ?btask .
-           ?btask   task:lead     ?who .
-           ?below   tass:employee ?managed .
-           ?managed empP:lastName  ?name .
-           ?managed empP:birthday  ?bday } 
-         ?who empP:lastName "Smith"^^xsd:string .
-         ?who empP:birthday ?bday }
--- a/sparql2sqlendpoint/src/main/resources/rdb2rdf.properties	Sun Oct 31 20:43:14 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,1 +0,0 @@
-default-stemuri = http://hr.example/DB/
\ No newline at end of file
--- a/sparql2sqlendpoint/src/main/scala/Config.scala	Sun Oct 31 20:43:14 2010 -0400
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,43 +0,0 @@
-package org.w3.sw.sparql2sqlendpoint
-
-import java.util.Properties
-import scala.io.{Source, Codec}
-
-import org.w3.sw.sql.SqlParser
-import org.w3.sw.rdb.RDB.Database
-
-trait ConfigHelper {
-
-  class P(p:Properties) {
-    def getProperty(key:String):Option[String] = 
-      try { Some(p.getProperty(key)) } catch {	case _ => None }
-  }
-  
-  def load(filename:String) = {
-    val prop = new Properties
-    prop.load(this.getClass.getResourceAsStream("/"+filename))
-    new P(prop)
-  }
-
-  def getContent(filename:String):String = {
-    val is = this.getClass.getResourceAsStream("/"+filename)
-    Source.fromInputStream(is)(Codec.UTF8).getLines().mkString("\n")
-  }
-
-}
-
-object Config extends ConfigHelper {
-
-  val rdb2rdfProp = load("rdb2rdf.properties")
-
-  val DDLParser = SqlParser()
-
-  val dbDdl = getContent("ddl.txt")
-
-  val db:org.w3.sw.rdb.RDB.Database = DDLParser.parseAll(DDLParser.ddl, dbDdl).get
-
-  val defaultSparqlQuery = getContent("default-sparql-query.txt")
-
-  val defaultStemURI = rdb2rdfProp.getProperty("default-stemuri").get
-
-}
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/sparql2sqlendpoint/src/main/scala/Configuration.scala	Wed Nov 03 14:03:52 2010 -0400
@@ -0,0 +1,105 @@
+package org.w3.util.fig
+
+import io.{Source, Codec}
+import java.io.File
+import net.liftweb.json._
+import net.liftweb.json.JsonAST._
+
+// adapted from http://github.com/codahale/fig
+
+object Configuration {
+  /**
+   * returns the content of a file, if accessible from the classpath
+   */
+  def fromFileInClasspath(filename:String):String = {
+    val inputStream = this.getClass.getResourceAsStream("/"+filename)
+    Source.fromInputStream(inputStream)(Codec.UTF8).getLines().mkString("\n")
+  }
+
+  /**
+   * maps the value to an Option. null or exception give None
+   */
+  def option[T](value : => T) : Option[T] = {
+    try {
+      Option(value)
+    } catch {
+      case _ => None
+    }
+  }
+
+}
+
+/**
+ * An exception class thrown when there is a configuration error.
+ */
+class ConfigurationException(message: String) extends Exception(message)
+
+/**
+ * A JSON-based configuration file. Line comments (i.e., //) are allowed.
+ *
+ * val config = new Configuration("config.json")
+ * config("rabbitmq.queue.name").as[String]
+ *
+ * @author coda
+ */
+class Configuration(filename: String) {
+  case class Value(path: String, value: JsonAST.JValue) {
+    /**
+     * Returns the value as an instance of type A.
+     */
+    def as[A](implicit mf: Manifest[A]) = value.extract[A](DefaultFormats, mf)
+
+    /**
+     * Returns the value as an instance of type Option[A]. If the value exists,
+     * Some(v: A) is returned; otherwise, None.
+     */
+    def asOption[A](implicit mf: Manifest[A]) = value.extractOpt[A](DefaultFormats, mf)
+
+    /**
+     * Returns the value as an instance of type A, or if the value does not
+     * exist, the result of the provided function.
+     */
+    def or[A](default: => A)(implicit mf: Manifest[A]) = asOption[A](mf).getOrElse(default)
+
+    /**
+     * Returns the value as an instance of type A, or if it cannot be converted,
+     * throws a ConfigurationException with an information error message.
+     */
+    def asRequired[A](implicit mf: Manifest[A]) = asOption[A] match {
+      case Some(v) => v
+      case None => throw new ConfigurationException(
+        "%s property %s not found".format(mf.erasure.getSimpleName, path)
+      )
+    }
+
+    /**
+     * Returns the value as a instance of List[A], or if the value is not a JSON
+     * array, an empty list.
+     */
+    def asList[A](implicit mf: Manifest[A]) = value match {
+      case JField(_, JArray(list)) => list.map { _.extract[A](DefaultFormats, mf) }
+      case other => List()
+    }
+
+    /**
+     * Returns the value as an instance of Map[String, A], or if the value is
+     * not a simple JSON object, an empty map.
+     */
+    def asMap[A](implicit mf: Manifest[A]) = value match {
+      case JField(_, o: JObject) =>
+        o.obj.map { f => f.name -> f.value.extract[A](DefaultFormats, mf) }.toMap
+      case other => Map()
+    }
+  }
+
+  private val json = {
+    val content = Configuration.option(Source.fromFile(new File(filename)).mkString) getOrElse Configuration.fromFileInClasspath(filename)
+    JsonParser.parse(content.replaceAll("""(^//.*|[\s]+//.*)""", ""))
+  }
+
+  /**
+   * Given a dot-notation JSON path (e.g., "parent.child.fieldname"), returns
+   * a Value which can be converted into a specific type or Option thereof.
+   */
+  def apply(path: String) = Value(path, path.split('.').foldLeft(json) { _ \ _ })
+}
--- a/sparql2sqlendpoint/src/main/scala/Servlet.scala	Sun Oct 31 20:43:14 2010 -0400
+++ b/sparql2sqlendpoint/src/main/scala/Servlet.scala	Wed Nov 03 14:03:52 2010 -0400
@@ -93,8 +93,53 @@
 
 }
 
+import org.w3.util.fig._
 
-class SparqlEndpoint extends HttpServlet {
+trait Config {
+
+  val config = new Configuration("default-config.json")
+  Class.forName(config("db.driver").as[String]).newInstance
+
+  def getConnection():Connection =
+    DriverManager.getConnection(config("db.url").as[String],
+                                config("config.user").as[String],
+                                config("config.password").as[String])
+  
+  val defaultStemURI:String = config("ui.stemuri").as[String]
+
+  val defaultSparqlQuery:String = config("ui.sparql-query").as[String]
+
+  val db:org.w3.sw.rdb.RDB.Database = {
+    val DDLParser = new org.w3.sw.sql.SqlParser()
+    val ddl = config("default.ddl").as[String]
+    println(ddl)
+    DDLParser.parseAll(DDLParser.ddl, ddl).get
+  }
+
+}
+
+object SPARQL2SQLEndpoint {
+
+  import org.eclipse.jetty.server.Server
+  import org.eclipse.jetty.servlet.{ServletContextHandler, ServletHolder}
+
+  /* see http://wiki.eclipse.org/Jetty/Tutorial/Embedding_Jetty */
+  def main(args: Array[String]) {
+
+    val server:Server = new Server(8080)
+    val context:ServletContextHandler = new ServletContextHandler(ServletContextHandler.SESSIONS)
+    context.setContextPath("/")
+    server.setHandler(context)
+  
+    context.addServlet(new ServletHolder(new SPARQL2SQLEndpoint()),"/*");
+    server.start
+    server.join
+
+  }
+
+}
+
+class SPARQL2SQLEndpoint extends HttpServlet with Config {
 
   val encoding = "utf-8"
 
@@ -109,18 +154,16 @@
 
   def processSparql(request:HttpServletRequest, response:HttpServletResponse, query:String) {
 
-    val stemURI:StemURI = StemURI(Some(request.getParameter("stemuri")) getOrElse Config.defaultStemURI)
+    val stemURI = new StemURI(Some(request.getParameter("stemuri")) getOrElse defaultStemURI)
 
     val sparqlParser = Sparql()
 
     val sparqlSelect = sparqlParser.parseAll(sparqlParser.select, query).get
 
-    val (generated, rdfmap) = SparqlToSql(Config.db, sparqlSelect, stemURI, true, false)
+    val (generated, rdfmap) = SparqlToSql(db, sparqlSelect, stemURI, true, false)
 
-    Class.forName("com.mysql.jdbc.Driver").newInstance
-    val connection:Connection = DriverManager.getConnection("jdbc:mysql://localhost/rdb2rdf", "root", "")
-
-//    if (! connection.isClosed) println("Successfully connected")
+    val connection:Connection = getConnection()
+    // if (! connection.isClosed) println("Successfully connected")
 
     val res = Control.process(connection=connection,
 			      sql=generated.toString,
@@ -145,8 +188,8 @@
           <h1>RDB2RDF Sparql endpoint</h1>
           <form action="sparql">
             <p>
-              StemURI: <input cols="80" name="stemuri" id="stemuri" value={ Config.defaultStemURI } /><br />
-              <textarea rows="10" cols="80" name="query" id="query">{ Config.defaultSparqlQuery }</textarea>
+              StemURI: <input cols="80" name="stemuri" id="stemuri" value={ defaultStemURI } /><br />
+              <textarea rows="10" cols="80" name="query" id="query">{ defaultSparqlQuery }</textarea>
               <input type="submit" />
             </p>
           </form>
--- a/sparql2sqlendpoint/src/main/webapp/WEB-INF/web.xml	Sun Oct 31 20:43:14 2010 -0400
+++ b/sparql2sqlendpoint/src/main/webapp/WEB-INF/web.xml	Wed Nov 03 14:03:52 2010 -0400
@@ -1,10 +1,10 @@
 <web-app xmlns="http://java.sun.com/xml/ns/javaee" version="2.5">
   <servlet>
-    <servlet-name>sparqlendpoint</servlet-name>
-    <servlet-class>org.w3.sparql2sql.servlet.SparqlEndpoint</servlet-class>
+    <servlet-name>sparql2sqlendpoint</servlet-name>
+    <servlet-class>org.w3.sw.sparql2sqlendpoint.SPARQL2SQLEndpoint</servlet-class>
   </servlet>
   <servlet-mapping>
-    <servlet-name>sparqlendpoint</servlet-name>
+    <servlet-name>sparql2sqlendpoint</servlet-name>
     <url-pattern>/sparql/*</url-pattern>
   </servlet-mapping>
 </web-app>