001package jmri.jmrit;
002
003import java.io.BufferedInputStream;
004import java.io.File;
005import java.io.FileInputStream;
006import java.io.FileNotFoundException;
007import java.io.FileOutputStream;
008import java.io.IOException;
009import java.io.InputStream;
010import java.net.URISyntaxException;
011import java.net.URL;
012import java.util.Calendar;
013import java.util.Date;
014import javax.annotation.Nonnull;
015import javax.swing.JFileChooser;
016import jmri.util.FileUtil;
017import jmri.util.JmriLocalEntityResolver;
018import jmri.util.NoArchiveFileFilter;
019import org.jdom2.Comment;
020import org.jdom2.Content;
021import org.jdom2.DocType;
022import org.jdom2.Document;
023import org.jdom2.Element;
024import org.jdom2.JDOMException;
025import org.jdom2.ProcessingInstruction;
026import org.jdom2.input.SAXBuilder;
027import org.jdom2.output.Format;
028import org.jdom2.output.XMLOutputter;
029import org.slf4j.Logger;
030import org.slf4j.LoggerFactory;
031
032/**
033 * Handle common aspects of XML files.
034 * <p>
035 * JMRI needs to be able to operate offline, so it needs to store resources
036 * locally. At the same time, we want XML files to be transportable, and to have
037 * their schema and stylesheets accessible via the web (for browser rendering).
038 * Further, our code assumes that default values for attributes will be
039 * provided, and it's necessary to read the schema for that to work.
040 * <p>
041 * We implement this using our own EntityResolver, the
042 * {@link jmri.util.JmriLocalEntityResolver} class.
043 * <p>
044 * When reading a file, validation is controlled heirarchically:
045 * <ul>
046 *   <li>There's a global default
047 *   <li>Which can be overridden on a particular XmlFile object
048 *   <li>Finally, the static call to create a builder can be invoked with a
049 * validation specification.
050 * </ul>
051 *
052 * @author Bob Jacobsen Copyright (C) 2001, 2002, 2007, 2012, 2014
053 */
054public class XmlFile {
055
056    /**
057     * Define root part of URL for XSLT style page processing instructions.
058     * <p>
059     * See the <A
060     * HREF="http://jmri.org/help/en/html/doc/Technical/XmlUsage.shtml#xslt">XSLT
061     * versioning discussion</a>.
062     * <p>
063     * Things that have been tried here: <dl>
064     * <dt>/xml/XSLT/ <dd>(Note leading slash) Works if there's a copy of the
065     * xml directory at the root of whatever served the XML file, e.g. the JMRI
066     * web site or a local computer running a server. Doesn't work for e.g.
067     * yahoo groups files. <dt>http://jmri.org/xml/XSLT/ <dd>Works well for
068     * files on the JMRI.org web server, but only that. </dl>
069     */
070    public static final String xsltLocation = "/xml/XSLT/";
071
072    /**
073     * Specify validation operations on input. The available choices are
074     * restricted to what the underlying SAX Xerces and JDOM implementations
075     * allow.
076     */
077    public enum Validate {
078        /**
079         * Don't validate input
080         */
081        None,
082        /**
083         * Require that the input specifies a Schema which validates
084         */
085        RequireSchema,
086        /**
087         * Validate against DTD if present (no DTD passes too)
088         */
089        CheckDtd,
090        /**
091         * Validate against DTD if present, else Schema must be present and
092         * valid
093         */
094        CheckDtdThenSchema
095    }
096
097    private String processingInstructionHRef;
098    private String processingInstructionType;
099
100    /**
101     * Get the value of the attribute 'href' of the process instruction of
102     * the last loaded document.
103     * @return the value of the attribute 'href' or null
104     */
105    public String getProcessingInstructionHRef() {
106        return processingInstructionHRef;
107    }
108
109    /**
110     * Get the value of the attribute 'type' of the process instruction of
111     * the last loaded document.
112     * @return the value of the attribute 'type' or null
113     */
114    public String getProcessingInstructionType() {
115        return processingInstructionType;
116    }
117
118    /**
119     * Read the contents of an XML file from its filename. The name is expanded
120     * by the {@link #findFile} routine. If the file is not found, attempts to
121     * read the XML file from a JAR resource.
122     *
123     * @param name Filename, as needed by {@link #findFile}
124     * @throws org.jdom2.JDOMException       only when all methods have failed
125     * @throws java.io.FileNotFoundException if file not found
126     * @return null if not found, else root element of located file
127     */
128    public Element rootFromName(String name) throws JDOMException, IOException {
129        File fp = findFile(name);
130        if (fp != null && fp.exists() && fp.canRead()) {
131            if (log.isDebugEnabled()) {
132                log.debug("readFile: {} from {}", name, fp.getAbsolutePath());
133            }
134            return rootFromFile(fp);
135        }
136        URL resource = FileUtil.findURL(name);
137        if (resource != null) {
138            return this.rootFromURL(resource);
139        } else {
140            if (!name.startsWith("xml")) {
141                return this.rootFromName("xml" + File.separator + name);
142            }
143            log.warn("Did not find file or resource {}", name);
144            throw new FileNotFoundException("Did not find file or resource " + name);
145        }
146    }
147
148    /**
149     * Read a File as XML, and return the root object.
150     * <p>
151     * Exceptions are only thrown when local recovery is impossible.
152     *
153     * @param file File to be parsed. A FileNotFoundException is thrown if it
154     *             doesn't exist.
155     * @throws org.jdom2.JDOMException       only when all methods have failed
156     * @throws java.io.FileNotFoundException if file not found
157     * @return root element from the file. This should never be null, as an
158     *         exception should be thrown if anything goes wrong.
159     */
160    public Element rootFromFile(File file) throws JDOMException, IOException {
161        if (log.isDebugEnabled()) {
162            log.debug("reading xml from file: {}", file.getPath());
163        }
164
165        try (FileInputStream fs = new FileInputStream(file)) {
166            return getRoot(fs);
167        }
168    }
169
170    /**
171     * Read an {@link java.io.InputStream} as XML, and return the root object.
172     * <p>
173     * Exceptions are only thrown when local recovery is impossible.
174     *
175     * @param stream InputStream to be parsed.
176     * @throws org.jdom2.JDOMException       only when all methods have failed
177     * @throws java.io.FileNotFoundException if file not found
178     * @return root element from the file. This should never be null, as an
179     *         exception should be thrown if anything goes wrong.
180     */
181    public Element rootFromInputStream(InputStream stream) throws JDOMException, IOException {
182        return getRoot(stream);
183    }
184
185    /**
186     * Read a URL as XML, and return the root object.
187     * <p>
188     * Exceptions are only thrown when local recovery is impossible.
189     *
190     * @param url URL locating the data file
191     * @throws org.jdom2.JDOMException only when all methods have failed
192     * @throws FileNotFoundException   if file not found
193     * @return root element from the file. This should never be null, as an
194     *         exception should be thrown if anything goes wrong.
195     */
196    public Element rootFromURL(URL url) throws JDOMException, IOException {
197        if (log.isDebugEnabled()) {
198            log.debug("reading xml from URL: {}", url.toString());
199        }
200        return getRoot(url.openConnection().getInputStream());
201    }
202
203    /**
204     * Get the root element from an XML document in a stream.
205     *
206     * @param stream input containing the XML document
207     * @return the root element of the XML document
208     * @throws org.jdom2.JDOMException if the XML document is invalid
209     * @throws java.io.IOException     if the input cannot be read
210     */
211    protected Element getRoot(InputStream stream) throws JDOMException, IOException {
212        log.trace("getRoot from stream");
213
214        processingInstructionHRef = null;
215        processingInstructionType = null;
216
217        SAXBuilder builder = getBuilder(getValidate());
218        Document doc = builder.build(new BufferedInputStream(stream));
219        doc = processInstructions(doc);  // handle any process instructions
220        // find root
221        return doc.getRootElement();
222    }
223
224    /**
225     * Write a File as XML.
226     *
227     * @param file File to be created.
228     * @param doc  Document to be written out. This should never be null.
229     * @throws FileNotFoundException if file not found
230     */
231    public void writeXML(File file, Document doc) throws IOException, FileNotFoundException {
232        // ensure parent directory exists
233        if (file.getParent() != null) {
234            FileUtil.createDirectory(file.getParent());
235        }
236        // write the result to selected file
237        try (FileOutputStream o = new FileOutputStream(file)) {
238            XMLOutputter fmt = new XMLOutputter();
239            fmt.setFormat(Format.getPrettyFormat()
240                    .setLineSeparator(System.getProperty("line.separator"))
241                    .setTextMode(Format.TextMode.TRIM_FULL_WHITE));
242            fmt.output(doc, o);
243            o.flush();
244        }
245    }
246
247    /**
248     * Check if a file of the given name exists. This uses the same search order
249     * as {@link #findFile}
250     *
251     * @param name file name, either absolute or relative
252     * @return true if the file exists in a searched place
253     */
254    protected boolean checkFile(String name) {
255        File fp = new File(name);
256        if (fp.exists()) {
257            return true;
258        }
259        fp = new File(FileUtil.getUserFilesPath() + name);
260        if (fp.exists()) {
261            return true;
262        } else {
263            File fx = new File(xmlDir() + name);
264            return fx.exists();
265        }
266    }
267
268    /**
269     * Get a File object for a name. This is here to implement the search
270     * rule:
271     * <ol>
272     *   <li>Look in user preferences directory, located by {@link jmri.util.FileUtil#getUserFilesPath()}
273     *   <li>Look in current working directory (usually the JMRI distribution directory)
274     *   <li>Look in program directory, located by {@link jmri.util.FileUtil#getProgramPath()}
275     *   <li>Look in XML directory, located by {@link #xmlDir}
276     *   <li>Check for absolute name.
277     * </ol>
278     *
279     * @param name Filename perhaps containing subdirectory information (e.g.
280     *             "decoders/Mine.xml")
281     * @return null if file found, otherwise the located File
282     */
283    protected File findFile(String name) {
284        URL url = FileUtil.findURL(name,
285                FileUtil.getUserFilesPath(),
286                ".",
287                FileUtil.getProgramPath(),
288                xmlDir());
289        if (url != null) {
290            try {
291                return new File(url.toURI());
292            } catch (URISyntaxException ex) {
293                return null;
294            }
295        }
296        return null;
297    }
298
299    /**
300     * Diagnostic printout of as much as we can find
301     *
302     * @param name Element to print, should not be null
303     */
304    static public void dumpElement(@Nonnull Element name) {
305        name.getChildren().forEach((element) -> {
306            log.info(" Element: {} ns: {}", element.getName(), element.getNamespace());
307        });
308    }
309
310    /**
311     * Move original file to a backup. Use this before writing out a new version
312     * of the file.
313     *
314     * @param name Last part of file pathname i.e. subdir/name, without the
315     *             pathname for either the xml or preferences directory.
316     */
317    public void makeBackupFile(String name) {
318        File file = findFile(name);
319        if (file == null) {
320            log.info("No {} file to backup", name);
321        } else if (file.canWrite()) {
322            String backupName = backupFileName(file.getAbsolutePath());
323            File backupFile = findFile(backupName);
324            if (backupFile != null) {
325                if (backupFile.delete()) {
326                    log.debug("deleted backup file {}", backupName);
327                }
328            }
329            if (file.renameTo(new File(backupName))) {
330                log.debug("created new backup file {}", backupName);
331            } else {
332                log.error("could not create backup file {}", backupName);
333            }
334        }
335    }
336
337    /**
338     * Move original file to backup directory.
339     *
340     * @param directory the backup directory to use.
341     * @param file      the file to be backed up. The file name will have the
342     *                  current date embedded in the backup name.
343     * @return true if successful.
344     */
345    public boolean makeBackupFile(String directory, File file) {
346        if (file == null) {
347            log.info("No file to backup");
348        } else if (file.canWrite()) {
349            String backupFullName = directory + File.separator + createFileNameWithDate(file.getName());
350            if (log.isDebugEnabled()) {
351                log.debug("new backup file: {}", backupFullName);
352            }
353
354            File backupFile = findFile(backupFullName);
355            if (backupFile != null) {
356                if (backupFile.delete()) {
357                    if (log.isDebugEnabled()) {
358                        log.debug("deleted backup file {}", backupFullName);
359                    }
360                }
361            } else {
362                backupFile = new File(backupFullName);
363            }
364            // create directory if needed
365            File parentDir = backupFile.getParentFile();
366            if (!parentDir.exists()) {
367                if (log.isDebugEnabled()) {
368                    log.debug("creating backup directory: {}", parentDir.getName());
369                }
370                if (!parentDir.mkdirs()) {
371                    log.error("backup directory not created");
372                    return false;
373                }
374            }
375            if (file.renameTo(new File(backupFullName))) {
376                if (log.isDebugEnabled()) {
377                    log.debug("created new backup file {}", backupFullName);
378                }
379            } else {
380                if (log.isDebugEnabled()) {
381                    log.debug("could not create backup file {}", backupFullName);
382                }
383                return false;
384            }
385        }
386        return true;
387    }
388
389    /**
390     * Revert to original file from backup. Use this for testing backup files.
391     *
392     * @param name Last part of file pathname i.e. subdir/name, without the
393     *             pathname for either the xml or preferences directory.
394     */
395    public void revertBackupFile(String name) {
396        File file = findFile(name);
397        if (file == null) {
398            log.info("No {} file to revert", name);
399        } else {
400            String backupName = backupFileName(file.getAbsolutePath());
401            File backupFile = findFile(backupName);
402            if (backupFile != null) {
403                log.info("No {} backup file to revert", backupName);
404                if (file.delete()) {
405                    log.debug("deleted original file {}", name);
406                }
407
408                if (backupFile.renameTo(new File(name))) {
409                    log.debug("created original file {}", name);
410                } else {
411                    log.error("could not create original file {}", name);
412                }
413            }
414        }
415    }
416
417    /**
418     * Return the name of a new, unique backup file. This is here so it can be
419     * overridden during tests. File to be backed-up must be within the
420     * preferences directory tree.
421     *
422     * @param name Filename without preference path information, e.g.
423     *             "decoders/Mine.xml".
424     * @return Complete filename, including path information into preferences
425     *         directory
426     */
427    public String backupFileName(String name) {
428        String f = name + ".bak";
429        if (log.isDebugEnabled()) {
430            log.debug("backup file name is: {}", f);
431        }
432        return f;
433    }
434
435    public String createFileNameWithDate(String name) {
436        // remove .xml extension
437        String[] fileName = name.split(".xml");
438        String f = fileName[0] + "_" + getDate() + ".xml";
439        if (log.isDebugEnabled()) {
440            log.debug("backup file name is: {}", f);
441        }
442        return f;
443    }
444
445    /**
446     * @return String based on the current date in the format of year month day
447     *         hour minute second. The date is fixed length and always returns a
448     *         date represented by 14 characters.
449     */
450    private String getDate() {
451        Calendar now = Calendar.getInstance();
452        return String.format("%d%02d%02d%02d%02d%02d",
453                now.get(Calendar.YEAR),
454                now.get(Calendar.MONTH) + 1,
455                now.get(Calendar.DATE),
456                now.get(Calendar.HOUR_OF_DAY),
457                now.get(Calendar.MINUTE),
458                now.get(Calendar.SECOND)
459        );
460    }
461
462    /**
463     * Execute the Processing Instructions in the file.
464     * <p>
465     * JMRI only knows about certain ones; the others will be ignored.
466     *
467     * @param doc the document containing processing instructions
468     * @return the processed document
469     */
470    Document processInstructions(Document doc) {
471        // this iterates over top level
472        for (Content c : doc.cloneContent()) {
473            if (c instanceof ProcessingInstruction) {
474                ProcessingInstruction pi = (ProcessingInstruction) c;
475                for (String attrName : pi.getPseudoAttributeNames()) {
476                    if ("href".equals(attrName)) {
477                        processingInstructionHRef = pi.getPseudoAttributeValue(attrName);
478                    }
479                    if ("type".equals(attrName)) {
480                        processingInstructionType = pi.getPseudoAttributeValue(attrName);
481                    }
482                }
483                try {
484                    doc = processOneInstruction((ProcessingInstruction) c, doc);
485                } catch (org.jdom2.transform.XSLTransformException ex) {
486                    log.error("XSLT error while transforming with {}, ignoring transform", c, ex);
487                } catch (org.jdom2.JDOMException ex) {
488                    log.error("JDOM error while transforming with {}, ignoring transform", c, ex);
489                } catch (java.io.IOException ex) {
490                    log.error("IO error while transforming with {}, ignoring transform", c, ex);
491                }
492            }
493        }
494
495        return doc;
496    }
497
498    Document processOneInstruction(ProcessingInstruction p, Document doc) throws org.jdom2.transform.XSLTransformException, org.jdom2.JDOMException, java.io.IOException {
499        log.trace("handling {}", p);
500
501        // check target
502        String target = p.getTarget();
503        if (!target.equals("transform-xslt")) {
504            return doc;
505        }
506
507        String href = p.getPseudoAttributeValue("href");
508        // we expect this to start with http://jmri.org/ and refer to the JMRI file tree
509        if (!href.startsWith("http://jmri.org/")) {
510            return doc;
511        }
512        href = href.substring(16);
513
514        // if starts with 'xml/' we remove that; findFile will put it back
515        if (href.startsWith("xml/")) {
516            href = href.substring(4);
517        }
518
519        // read the XSLT transform into a Document to get XInclude done
520        SAXBuilder builder = getBuilder(Validate.None);
521        Document xdoc = builder.build(new BufferedInputStream(new FileInputStream(findFile(href))));
522        org.jdom2.transform.XSLTransformer transformer = new org.jdom2.transform.XSLTransformer(xdoc);
523        return transformer.transform(doc);
524    }
525
526    /**
527     * Create the Document object to store a particular root Element.
528     *
529     * @param root Root element of the final document
530     * @param dtd  name of an external DTD
531     * @return new Document, with root installed
532     */
533    static public Document newDocument(Element root, String dtd) {
534        Document doc = new Document(root);
535        doc.setDocType(new DocType(root.getName(), dtd));
536        addDefaultInfo(root);
537        return doc;
538    }
539
540    /**
541     * Create the Document object to store a particular root Element, without a
542     * DocType DTD (e.g. for using a schema)
543     *
544     * @param root Root element of the final document
545     * @return new Document, with root installed
546     */
547    static public Document newDocument(Element root) {
548        Document doc = new Document(root);
549        addDefaultInfo(root);
550        return doc;
551    }
552
553    /**
554     * Add default information to the XML before writing it out.
555     * <p>
556     * Currently, this is identification information as an XML comment. This
557     * includes: <ul>
558     * <li>The JMRI version used <li>Date of writing <li>A CVS id string, in
559     * case the file gets checked in or out </ul>
560     * <p>
561     * It may be necessary to extend this to check whether the info is already
562     * present, e.g. if re-writing a file.
563     *
564     * @param root The root element of the document that will be written.
565     */
566    static public void addDefaultInfo(Element root) {
567        String content = "Written by JMRI version " + jmri.Version.name()
568                + " on " + (new Date()).toString();
569        Comment comment = new Comment(content);
570        root.addContent(comment);
571    }
572
573    /**
574     * Define the location of XML files within the distribution directory.
575     * <p>
576     * Use {@link FileUtil#getProgramPath()} since the current working directory
577     * is not guaranteed to be the JMRI distribution directory if jmri.jar is
578     * referenced by an external Java application.
579     *
580     * @return the XML directory that ships with JMRI.
581     */
582    static public String xmlDir() {
583        return FileUtil.getProgramPath() + "xml" + File.separator;
584    }
585
586    /**
587     * Whether to, by global default, validate the file being read. Public so it
588     * can be set by scripting and for debugging.
589     *
590     * @return the default level of validation to apply to a file
591     */
592    static public Validate getDefaultValidate() {
593        return defaultValidate;
594    }
595
596    static public void setDefaultValidate(Validate v) {
597        defaultValidate = v;
598    }
599
600    static private Validate defaultValidate = Validate.None;
601
602    /**
603     * Whether to verify the DTD of this XML file when read.
604     *
605     * @return the level of validation to apply to a file
606     */
607    public Validate getValidate() {
608        return validate;
609    }
610
611    public void setValidate(Validate v) {
612        validate = v;
613    }
614
615    private Validate validate = defaultValidate;
616
617    /**
618     * Get the default standard location for DTDs in new XML documents. Public
619     * so it can be set by scripting and for debug.
620     *
621     * @return the default DTD location
622     */
623    static public String getDefaultDtdLocation() {
624        return defaultDtdLocation;
625    }
626
627    static public void setDefaultDtdLocation(String v) {
628        defaultDtdLocation = v;
629    }
630
631    static String defaultDtdLocation = "/xml/DTD/";
632
633    /**
634     * Get the location for DTDs in this XML document.
635     *
636     * @return the DTD location
637     */
638    public String getDtdLocation() {
639        return dtdLocation;
640    }
641
642    public void setDtdLocation(String v) {
643        dtdLocation = v;
644    }
645
646    public String dtdLocation = defaultDtdLocation;
647
648    /**
649     * Provide a JFileChooser initialized to the default user location, and with
650     * a default filter. This filter excludes {@code .zip} and {@code .jar}
651     * archives.
652     *
653     * @param filter Title for the filter, may not be null
654     * @param suffix Allowed file extensions, if empty all extensions are
655     *               allowed except {@code .zip} and {@code .jar}; include an
656     *               empty String to allow files without an extension if
657     *               specifying other extensions.
658     * @return a file chooser
659     */
660    public static JFileChooser userFileChooser(String filter, String... suffix) {
661        JFileChooser fc = new jmri.util.swing.JmriJFileChooser(FileUtil.getUserFilesPath());
662        fc.setFileFilter(new NoArchiveFileFilter(filter, suffix));
663        return fc;
664    }
665
666    /**
667     * Provide a JFileChooser initialized to the default user location, and with
668     * a default filter. This filter excludes {@code .zip} and {@code .jar}
669     * archives.
670     *
671     * @return a file chooser
672     */
673    public static JFileChooser userFileChooser() {
674        JFileChooser fc = new jmri.util.swing.JmriJFileChooser(FileUtil.getUserFilesPath());
675        fc.setFileFilter(new NoArchiveFileFilter());
676        return fc;
677    }
678
679    @SuppressWarnings("deprecation") // org.jdom2.input.SAXBuilder(java.lang.String saxDriverClass, boolean validate)
680    //{@see http://www.jdom.org/docs/apidocs/org/jdom2/input/SAXBuilder.html}
681    //{@see http://www.jdom.org/docs/apidocs/org/jdom2/input/sax/XMLReaders.html#NONVALIDATING}
682    // Validate.CheckDtdThenSchema may not be available readily
683    public static SAXBuilder getBuilder(Validate validate) {  // should really be a Verify enum
684        SAXBuilder builder;
685
686        boolean verifyDTD = (validate == Validate.CheckDtd) || (validate == Validate.CheckDtdThenSchema);
687        boolean verifySchema = (validate == Validate.RequireSchema) || (validate == Validate.CheckDtdThenSchema);
688
689        // old style
690        builder = new SAXBuilder("org.apache.xerces.parsers.SAXParser", verifyDTD);  // argument controls DTD validation
691
692        // insert local resolver for includes, schema, DTDs
693        builder.setEntityResolver(new JmriLocalEntityResolver());
694
695        // configure XInclude handling
696        builder.setFeature("http://apache.org/xml/features/xinclude", true);
697        builder.setFeature("http://apache.org/xml/features/xinclude/fixup-base-uris", false);
698
699        // only validate if grammar is available, making ABSENT OK
700        builder.setFeature("http://apache.org/xml/features/validation/dynamic", verifyDTD && !verifySchema);
701
702        // control Schema validation
703        builder.setFeature("http://apache.org/xml/features/validation/schema", verifySchema);
704        builder.setFeature("http://apache.org/xml/features/validation/schema-full-checking", verifySchema);
705
706        // if not validating DTD, just validate Schema
707        builder.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", verifyDTD);
708        if (!verifyDTD) {
709            builder.setProperty("http://java.sun.com/xml/jaxp/properties/schemaLanguage", "http://www.w3.org/2001/XMLSchema");
710        }
711
712        // allow Java character encodings
713        builder.setFeature("http://apache.org/xml/features/allow-java-encodings", true);
714
715        return builder;
716    }
717
718    // initialize logging
719    private static final Logger log = LoggerFactory.getLogger(XmlFile.class);
720
721}