1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25 package org.utgenome.gwt.utgb.server;
26
27 import java.io.BufferedReader;
28 import java.io.BufferedWriter;
29 import java.io.File;
30 import java.io.FileReader;
31 import java.io.IOException;
32 import java.io.InputStreamReader;
33 import java.io.StringReader;
34 import java.io.StringWriter;
35 import java.net.URL;
36 import java.util.ArrayList;
37 import java.util.Iterator;
38 import java.util.List;
39
40 import javax.servlet.http.HttpServletRequest;
41 import javax.servlet.http.HttpSession;
42
43 import net.sf.samtools.SAMFileReader;
44 import net.sf.samtools.SAMRecord;
45
46 import org.utgenome.UTGBException;
47 import org.utgenome.format.fasta.CompactFASTA;
48 import org.utgenome.format.keyword.KeywordDB;
49 import org.utgenome.format.sam.SAM2SilkReader;
50 import org.utgenome.format.wig.WIGDatabaseReader;
51 import org.utgenome.gwt.utgb.client.BrowserService;
52 import org.utgenome.gwt.utgb.client.UTGBClientErrorCode;
53 import org.utgenome.gwt.utgb.client.UTGBClientException;
54 import org.utgenome.gwt.utgb.client.bean.DatabaseEntry;
55 import org.utgenome.gwt.utgb.client.bio.ChrLoc;
56 import org.utgenome.gwt.utgb.client.bio.ChrRange;
57 import org.utgenome.gwt.utgb.client.bio.CompactWIGData;
58 import org.utgenome.gwt.utgb.client.bio.Gene;
59 import org.utgenome.gwt.utgb.client.bio.GenomeDB;
60 import org.utgenome.gwt.utgb.client.bio.GraphWindow;
61 import org.utgenome.gwt.utgb.client.bio.Interval;
62 import org.utgenome.gwt.utgb.client.bio.KeywordSearchResult;
63 import org.utgenome.gwt.utgb.client.bio.OnGenome;
64 import org.utgenome.gwt.utgb.client.bio.ReadQueryConfig;
65 import org.utgenome.gwt.utgb.client.bio.SAMRead;
66 import org.utgenome.gwt.utgb.client.bio.WigGraphData;
67 import org.utgenome.gwt.utgb.client.track.bean.TrackBean;
68 import org.utgenome.gwt.utgb.client.view.TrackView;
69 import org.utgenome.gwt.utgb.server.app.ChromosomeMap;
70 import org.utgenome.gwt.utgb.server.app.ReadView;
71 import org.utgenome.gwt.utgb.server.util.WebApplicationResource;
72 import org.xerial.core.XerialException;
73 import org.xerial.db.DBException;
74 import org.xerial.db.sql.SQLExpression;
75 import org.xerial.db.sql.sqlite.SQLiteAccess;
76 import org.xerial.db.sql.sqlite.SQLiteCatalog;
77 import org.xerial.json.JSONArray;
78 import org.xerial.json.JSONObject;
79 import org.xerial.lens.JSONLens;
80 import org.xerial.lens.SilkLens;
81 import org.xerial.lens.XMLLens;
82 import org.xerial.util.ObjectHandlerBase;
83 import org.xerial.util.StopWatch;
84 import org.xerial.util.log.Logger;
85
86 import com.google.gwt.user.server.rpc.RemoteServiceServlet;
87
88
89
90
91
92
93
94
95
96 public class BrowserServiceImpl extends RemoteServiceServlet implements BrowserService {
97
98 static Logger _logger = Logger.getLogger(BrowserServiceImpl.class);
99
100
101
102
103 private static final long serialVersionUID = 1L;
104
105 public BrowserServiceImpl() throws DBException {
106
107 }
108
109 private static String sanitizeViewName(String name) {
110 return name.replaceAll("\\.\\.", "");
111 }
112
113 public TrackView createTrackView(String silk) throws UTGBClientException {
114 try {
115 TrackView v = SilkLens.loadSilk(TrackView.class, new StringReader(silk));
116 return v;
117 }
118 catch (Exception e) {
119 throw new UTGBClientException(UTGBClientErrorCode.PARSE_ERROR, "parse error: " + e.getMessage());
120 }
121 }
122
123 public TrackView getTrackView(String viewName) throws UTGBClientException {
124
125 if (viewName.startsWith("http://")) {
126 _logger.info(String.format("loading view: ", viewName));
127 String view = getHTTPContent(viewName);
128 try {
129 TrackView v = SilkLens.loadSilk(TrackView.class, new URL(viewName));
130 return v;
131 }
132 catch (IOException e) {
133 throw new UTGBClientException(UTGBClientErrorCode.MISSING_FILES, "failed to retrieve view from " + viewName);
134 }
135 catch (XerialException e) {
136 throw new UTGBClientException(UTGBClientErrorCode.PARSE_ERROR, "parse error: " + e.getMessage());
137 }
138 }
139 else {
140 File viewFile = new File("config/view", sanitizeViewName(viewName) + ".silk");
141 try {
142
143 FileReader f = null;
144 try {
145 File viewFilePath = new File(UTGBMaster.getProjectRootFolder(), viewFile.getPath());
146 _logger.info(String.format("loading view:" + viewFile));
147 if (!viewFilePath.exists())
148 throw new UTGBClientException(UTGBClientErrorCode.MISSING_FILES, String.format("%s is not found", viewFile));
149
150 f = new FileReader(viewFilePath);
151 TrackView v = SilkLens.loadSilk(TrackView.class, f);
152 return v;
153 }
154 catch (UTGBException e) {
155 throw new UTGBClientException(UTGBClientErrorCode.NOT_IN_PROJECT_ROOT, String.format("not in the project root"));
156 }
157 catch (XerialException e) {
158 throw new UTGBClientException(UTGBClientErrorCode.PARSE_ERROR, String.format("parse error (%s): ", e));
159 }
160 finally {
161 if (f != null)
162 f.close();
163 }
164 }
165 catch (IOException e) {
166 throw new UTGBClientException(UTGBClientErrorCode.IO_ERROR, String.format("failed to close vilew file %s: %s", viewFile, e));
167 }
168 }
169 }
170
171 public String getDataModelParameter(String amoebaQuery) {
172 return null;
173 }
174
175 public String getHTTPContent(String url) {
176 try {
177
178 BufferedReader in = openURL(url);
179
180 StringWriter buffer = new StringWriter();
181 BufferedWriter out = new BufferedWriter(buffer);
182 String line;
183 while ((line = in.readLine()) != null) {
184 out.append(line);
185 out.newLine();
186 }
187 in.close();
188 out.flush();
189 return buffer.toString();
190 }
191 catch (IOException e) {
192 _logger.error(e);
193 return "";
194 }
195 }
196
197 public String getDatabaseCatalog(String jdbcAddress) {
198 try {
199
200 SQLiteAccess dbAccess = new SQLiteAccess(jdbcAddress);
201 SQLiteCatalog catalog = dbAccess.getCatalog();
202 String json = catalog.toJSON();
203 _logger.debug("catalog: " + json);
204 return json;
205 }
206 catch (DBException e) {
207 _logger.error(e);
208 return "";
209 }
210 }
211
212 public String getTableData(String jdbcAddress, String tableName) {
213 try {
214 SQLiteAccess dbAccess = new SQLiteAccess(jdbcAddress);
215 String sql = SQLExpression.fillTemplate("select * from $1", tableName);
216 List<JSONObject> result = dbAccess.jsonQuery(sql);
217 JSONObject resultObj = new JSONObject();
218 JSONArray resultArray = new JSONArray();
219 for (JSONObject row : result) {
220 resultArray.add(row);
221 }
222 resultObj.put("data", resultArray);
223
224 String json = resultObj.toJSONString();
225 _logger.debug("query result: " + json);
226 return json;
227 }
228 catch (DBException e) {
229 _logger.error(e);
230 return "";
231 }
232 }
233
234 public List<TrackBean> getTrackList(int entriesPerPage, int page) {
235 ArrayList<TrackBean> trackList = new ArrayList<TrackBean>();
236 ArrayList<String> trackXMLFileList = WebApplicationResource.find(this.getServletContext(), "/tracks/", "(.*)\\.xml$", true);
237 for (String xmlPath : trackXMLFileList) {
238 try {
239 trackList.add(loadTrackInfo(xmlPath));
240 }
241 catch (UTGBException e) {
242 _logger.error(e);
243 }
244 }
245
246 return trackList;
247 }
248
249 private TrackBean loadTrackInfo(String trackXMLResourcePath) throws UTGBException {
250 try {
251 BufferedReader xmlReader = WebApplicationResource.openResource(this.getServletContext(), trackXMLResourcePath);
252 TrackBean trackInfo = XMLLens.createXMLBean(TrackBean.class, xmlReader);
253 return trackInfo;
254 }
255 catch (XerialException e) {
256 throw new UTGBException(e);
257 }
258 catch (IOException e) {
259 throw new UTGBException(e);
260 }
261 }
262
263 public List<TrackBean> getTrackList(String prefix, int entriesPerPage, int page) {
264
265 return null;
266 }
267
268 public int numHitsOfTracks(String prefix) {
269
270 return 0;
271 }
272
273 public KeywordSearchResult keywordSearch(String species, String revision, String keyword, int numEntriesPerPage, int page) throws UTGBClientException {
274
275 File keywordDBFile = new File(WebTrackBase.getProjectRootPath(), "db/keyword.sqlite");
276 KeywordDB db = null;
277 try {
278 db = new KeywordDB(keywordDBFile);
279 return db.query(revision, keyword, page, numEntriesPerPage);
280 }
281 catch (Exception e) {
282 e.printStackTrace();
283 throw new UTGBClientException(e.getMessage());
284 }
285 finally {
286 if (db != null)
287 try {
288 db.close();
289 }
290 catch (DBException e) {
291 throw new UTGBClientException(e.getMessage());
292 }
293 }
294
295 }
296
297 public void saveView(String viewXML) {
298 HttpServletRequest request = this.getThreadLocalRequest();
299 HttpSession session = request.getSession(true);
300 session.setAttribute("view", viewXML);
301 }
302
303 public String getCurrentView() {
304 HttpServletRequest request = this.getThreadLocalRequest();
305 HttpSession session = request.getSession(true);
306 _logger.debug("session ID = " + session.getId());
307 String viewXML = (String) session.getAttribute("view");
308 if (viewXML == null || (viewXML != null && viewXML.length() == 0)) {
309 try {
310 String xml = WebApplicationResource.getContent(this.getServletContext(), "/view/standard.xml");
311 return xml;
312 }
313 catch (IOException e) {
314 _logger.error(e);
315 return null;
316 }
317 }
318 return viewXML;
319 }
320
321 private BufferedReader openURL(String url) throws IOException {
322 BufferedReader in;
323 if (!url.startsWith("http://")) {
324 if (!url.startsWith("/"))
325 url = "/" + url;
326 _logger.debug("proxy request: " + url);
327 in = WebApplicationResource.openResource(this.getServletContext(), url);
328 }
329 else {
330 URL address = new URL(url);
331 _logger.debug("proxy request: " + url);
332 in = new BufferedReader(new InputStreamReader(address.openStream()));
333 }
334 return in;
335 }
336
337 static class BeanRetriever<T> extends ObjectHandlerBase<T> {
338 private ArrayList<T> geneList = new ArrayList<T>();
339
340 public BeanRetriever() {
341 }
342
343 public ArrayList<T> getResult() {
344 return geneList;
345 }
346
347 public void handle(T bean) throws Exception {
348 geneList.add(bean);
349 }
350
351 public void handleException(Exception e) throws Exception {
352 _logger.error(e);
353 }
354 }
355
356 static class RefseqGeneRetriever extends BeanRetriever<Gene> {
357
358 @Override
359 public void handle(Gene g) throws Exception {
360 g.adjustToOneOrigin();
361 super.handle(g);
362 }
363
364 }
365
366 public List<Gene> getGeneList(String serviceURI) {
367
368 StopWatch sw = new StopWatch();
369 try {
370 BufferedReader reader = openURL(serviceURI);
371
372 RefseqGeneRetriever geneRetriever = new RefseqGeneRetriever();
373 JSONLens.findFromJSON(reader, "gene", Gene.class, geneRetriever);
374 return geneRetriever.getResult();
375 }
376 catch (Exception e) {
377 _logger.error(e);
378 }
379 finally {
380 _logger.debug("proxy request: done " + sw.getElapsedTime() + " sec");
381 }
382 return new ArrayList<Gene>();
383 }
384
385 public ChrRange getChrRegion(String species, String revision) {
386 return ChromosomeMap.getChrRegion(species, revision);
387 }
388
389 public List<Interval> getLocusList(String dbGroup, String dbName, ChrLoc location) {
390
391 String bssFolder = UTGBMaster.getUTGBConfig().getProperty("utgb.db.folder", WebTrackBase.getProjectRootPath() + "/db");
392 File dbFile = new File(bssFolder, String.format("%s/%s", dbGroup, dbName));
393
394 if (dbFile.exists()) {
395 try {
396 SQLiteAccess dbAccess = new SQLiteAccess(dbFile.getAbsolutePath());
397 String sql = WebTrackBase
398 .createSQLStatement(
399 "select rowid, queryID as name, targetStart as start, targetEnd as end, strand from alignment where target = '$1' and start between $2 - $4 and $3 + $4 and (start <= $3 or end >= $2) order by start, end-start desc",
400 location.chr, location.start, location.end, 1000);
401
402
403 List<Interval> result = dbAccess.query(sql, Interval.class);
404
405 return result;
406 }
407 catch (Exception e) {
408 _logger.error(e);
409 }
410 }
411
412 return new ArrayList<Interval>();
413 }
414
415 public List<String> getChildDBGroups(String parentDBGroup) {
416
417 String dbFolder = UTGBMaster.getUTGBConfig().getProperty("utgb.db.folder", WebTrackBase.getProjectRootPath() + "/db");
418 File parentFolder = new File(dbFolder, parentDBGroup);
419
420 ArrayList<String> result = new ArrayList<String>();
421 if (parentFolder.exists()) {
422 File[] ls = parentFolder.listFiles();
423 if (ls != null) {
424 for (File each : ls) {
425 if (each.isDirectory()) {
426 if (each.getName().startsWith("."))
427 continue;
428 result.add(parentDBGroup + "/" + each.getName());
429 }
430 }
431 }
432 }
433
434 return result;
435 }
436
437 public List<String> getDBNames(String dbGroup) {
438
439 String dbFolder = UTGBMaster.getUTGBConfig().getProperty("utgb.db.folder", WebTrackBase.getProjectRootPath() + "/db");
440 File groupFolder = new File(dbFolder, dbGroup);
441
442 ArrayList<String> result = new ArrayList<String>();
443 if (groupFolder.exists()) {
444 File[] ls = groupFolder.listFiles();
445 if (ls != null) {
446 for (File each : ls) {
447 if (each.isFile()) {
448 if (each.getName().startsWith("."))
449 continue;
450 result.add(each.getName());
451 }
452 }
453 }
454 }
455
456 return result;
457 }
458
459 public List<DatabaseEntry> getDBEntry(String dbGroup) {
460
461 String dbFolder = UTGBMaster.getUTGBConfig().getProperty("utgb.db.folder", WebTrackBase.getProjectRootPath() + "/db");
462 File groupFolder = new File(dbFolder, dbGroup);
463
464 ArrayList<DatabaseEntry> result = new ArrayList<DatabaseEntry>();
465 if (groupFolder.exists()) {
466 File[] ls = groupFolder.listFiles();
467 if (ls != null) {
468 for (File each : ls) {
469 if (each.getName().startsWith("."))
470 continue;
471 if (each.isFile()) {
472 result.add(DatabaseEntry.newFile(each.getName()));
473 }
474 else if (each.isDirectory()) {
475 result.add(DatabaseEntry.newFolder(each.getName()));
476 }
477 }
478 }
479 }
480
481 return result;
482 }
483
484 public List<WigGraphData> getWigDataList(String fileName, int windowWidth, ChrLoc location) {
485 List<WigGraphData> wigDataList = null;
486
487 if (!ReadView.isDescendant(fileName)) {
488 _logger.error("path must be under the project root: " + fileName);
489 return new ArrayList<WigGraphData>();
490 }
491
492 try {
493 wigDataList = WIGDatabaseReader.getWigDataList(new File(WebTrackBase.getProjectRootPath(), fileName), windowWidth, location, GraphWindow.MAX);
494 }
495 catch (Exception e) {
496 _logger.error(e);
497 e.printStackTrace(System.err);
498 }
499
500 return wigDataList;
501 }
502
503 public List<CompactWIGData> getCompactWigDataList(String path, int windowWidth, ChrLoc location) {
504
505 if (!ReadView.isDescendant(path)) {
506 _logger.error("path must be under the project root: " + path);
507 return new ArrayList<CompactWIGData>();
508 }
509
510 List<CompactWIGData> cWig = null;
511 try {
512 cWig = WIGDatabaseReader.getCompactWigDataList(new File(WebTrackBase.getProjectRootPath(), path), windowWidth, location, GraphWindow.MAX);
513 }
514 catch (Exception e) {
515 _logger.error(e);
516 e.printStackTrace(System.err);
517 }
518
519 return cWig;
520 }
521
522 public List<SAMRead> getSAMReadList(String readFileName, String refSeqFileName) {
523 final ArrayList<SAMRead> readDataList = new ArrayList<SAMRead>();
524
525 try {
526 _logger.info(WebTrackBase.getProjectRootPath() + "/" + readFileName);
527 _logger.info(WebTrackBase.getProjectRootPath() + "/" + refSeqFileName);
528 final CompactFASTA cf = new CompactFASTA(WebTrackBase.getProjectRootPath() + "/" + refSeqFileName);
529
530 SilkLens.findFromSilk(new SAM2SilkReader(new FileReader(new File(WebTrackBase.getProjectRootPath() + "/" + readFileName))), "record",
531 SAMRead.class, new ObjectHandlerBase<SAMRead>() {
532 public void handle(SAMRead input) throws Exception {
533 input.refSeq = cf.getSequence(input.rname, input.getStart() - 1, input.getEnd()).toString();
534 _logger.info(SilkLens.toSilk(input));
535 readDataList.add(input);
536 }
537 });
538 }
539 catch (Exception e) {
540 _logger.error(e);
541 }
542 return readDataList;
543 }
544
545 public List<SAMRead> querySAMReadList(String bamFileName, String indexFileName, String refSeqFileName, String rname, int start, int end) {
546 final ArrayList<SAMRead> readDataList = new ArrayList<SAMRead>();
547
548 try {
549 _logger.info(WebTrackBase.getProjectRootPath() + "/" + bamFileName);
550 _logger.info(WebTrackBase.getProjectRootPath() + "/" + indexFileName);
551 _logger.info(WebTrackBase.getProjectRootPath() + "/" + refSeqFileName);
552 final CompactFASTA cf = new CompactFASTA(WebTrackBase.getProjectRootPath() + "/" + refSeqFileName);
553
554 SAMFileReader reader = new SAMFileReader(new File(WebTrackBase.getProjectRootPath() + "/" + bamFileName), new File(
555 WebTrackBase.getProjectRootPath() + "/" + indexFileName));
556
557 StopWatch st1 = new StopWatch();
558 Iterator<SAMRecord> iterator = reader.query(rname, start, end, true);
559 _logger.info("st1:" + st1.getElapsedTime());
560
561 while (iterator.hasNext()) {
562 SAMRecord record = iterator.next();
563 _logger.info(record.format());
564
565
566
567 SAMRead read = SAM2SilkReader.convertToSAMRead(record);
568
569 read.refSeq = cf.getSequence(read.rname, read.getStart() - 1, read.getEnd()).toString();
570
571 readDataList.add(read);
572 }
573 }
574 catch (Exception e) {
575 _logger.error(e);
576 }
577 return readDataList;
578 }
579
580 public String getRefSeq(String refSeqFileName, String rname, int start, int end) {
581 _logger.info(rname + ":" + start + "-" + end);
582 String refSeq = null;
583 try {
584 _logger.info(WebTrackBase.getProjectRootPath() + "/" + refSeqFileName);
585 final CompactFASTA cf = new CompactFASTA(WebTrackBase.getProjectRootPath() + "/" + refSeqFileName);
586
587 if (end - start > 10000)
588 end = start + 10000;
589
590 refSeq = cf.getSequence(rname, start - 1, end).toString();
591 _logger.info(refSeq);
592 }
593 catch (Exception e) {
594 _logger.error(e);
595 }
596 return refSeq;
597 }
598
599 public List<OnGenome> getOnGenomeData(GenomeDB db, ChrLoc range, ReadQueryConfig config) {
600 return ReadView.overlapQuery(db, range, config);
601 }
602
603 }