Source code

001package jmri.jmrit;
002
003import java.io.BufferedInputStream;
004import java.io.File;
005import java.io.FileInputStream;
006import java.io.FileNotFoundException;
007import java.io.FileOutputStream;
008import java.io.IOException;
009import java.io.InputStream;
010import java.net.URISyntaxException;
011import java.net.URL;
012import java.util.Calendar;
013import java.util.Date;
014
015import javax.annotation.Nonnull;
016import javax.swing.JFileChooser;
017
018import jmri.InstanceManager;
019import jmri.configurexml.LoadAndStorePreferences;
020import jmri.util.FileUtil;
021import jmri.util.JmriLocalEntityResolver;
022import jmri.util.NoArchiveFileFilter;
023
024import org.jdom2.Comment;
025import org.jdom2.Content;
026import org.jdom2.DocType;
027import org.jdom2.Document;
028import org.jdom2.Element;
029import org.jdom2.JDOMException;
030import org.jdom2.ProcessingInstruction;
031import org.jdom2.input.SAXBuilder;
032import org.jdom2.output.Format;
033import org.jdom2.output.XMLOutputter;
034import org.slf4j.Logger;
035import org.slf4j.LoggerFactory;
036
037/**
038 * Handle common aspects of XML files.
039 * <p>
040 * JMRI needs to be able to operate offline, so it needs to store resources
041 * locally. At the same time, we want XML files to be transportable, and to have
042 * their schema and stylesheets accessible via the web (for browser rendering).
043 * Further, our code assumes that default values for attributes will be
044 * provided, and it's necessary to read the schema for that to work.
045 * <p>
046 * We implement this using our own EntityResolver, the
047 * {@link jmri.util.JmriLocalEntityResolver} class.
048 * <p>
049 * When reading a file, validation is controlled heirarchically:
050 * <ul>
051 *   <li>There's a global default
052 *   <li>Which can be overridden on a particular XmlFile object
053 *   <li>Finally, the static call to create a builder can be invoked with a
054 * validation specification.
055 * </ul>
056 *
057 * @author Bob Jacobsen Copyright (C) 2001, 2002, 2007, 2012, 2014
058 */
059public class XmlFile {
060
061    /**
062     * Define root part of URL for XSLT style page processing instructions.
063     * <p>
064     * See the <A
065     * HREF="http://jmri.org/help/en/html/doc/Technical/XmlUsage.shtml#xslt">XSLT
066     * versioning discussion</a>.
067     * <p>
068     * Things that have been tried here: <dl>
069     * <dt>/xml/XSLT/ <dd>(Note leading slash) Works if there's a copy of the
070     * xml directory at the root of whatever served the XML file, e.g. the JMRI
071     * web site or a local computer running a server. Doesn't work for e.g.
072     * yahoo groups files. <dt>http://jmri.org/xml/XSLT/ <dd>Works well for
073     * files on the JMRI.org web server, but only that. </dl>
074     */
075    public static final String xsltLocation = "/xml/XSLT/";
076
077    /**
078     * Specify validation operations on input. The available choices are
079     * restricted to what the underlying SAX Xerces and JDOM implementations
080     * allow.
081     */
082    public enum Validate {
083        /**
084         * Don't validate input
085         */
086        None,
087        /**
088         * Require that the input specifies a Schema which validates
089         */
090        RequireSchema,
091        /**
092         * Validate against DTD if present (no DTD passes too)
093         */
094        CheckDtd,
095        /**
096         * Validate against DTD if present, else Schema must be present and
097         * valid
098         */
099        CheckDtdThenSchema
100    }
101
102    private String processingInstructionHRef;
103    private String processingInstructionType;
104
105    /**
106     * Get the value of the attribute 'href' of the process instruction of
107     * the last loaded document.
108     * @return the value of the attribute 'href' or null
109     */
110    public String getProcessingInstructionHRef() {
111        return processingInstructionHRef;
112    }
113
114    /**
115     * Get the value of the attribute 'type' of the process instruction of
116     * the last loaded document.
117     * @return the value of the attribute 'type' or null
118     */
119    public String getProcessingInstructionType() {
120        return processingInstructionType;
121    }
122
123    /**
124     * Read the contents of an XML file from its filename. The name is expanded
125     * by the {@link #findFile} routine. If the file is not found, attempts to
126     * read the XML file from a JAR resource.
127     *
128     * @param name Filename, as needed by {@link #findFile}
129     * @throws org.jdom2.JDOMException       only when all methods have failed
130     * @throws java.io.FileNotFoundException if file not found
131     * @return null if not found, else root element of located file
132     */
133    public Element rootFromName(String name) throws JDOMException, IOException {
134        File fp = findFile(name);
135        if (fp != null && fp.exists() && fp.canRead()) {
136            if (log.isDebugEnabled()) {
137                log.debug("readFile: {} from {}", name, fp.getAbsolutePath());
138            }
139            return rootFromFile(fp);
140        }
141        URL resource = FileUtil.findURL(name);
142        if (resource != null) {
143            return this.rootFromURL(resource);
144        } else {
145            if (!name.startsWith("xml")) {
146                return this.rootFromName("xml" + File.separator + name);
147            }
148            log.warn("Did not find file or resource {}", name);
149            throw new FileNotFoundException("Did not find file or resource " + name);
150        }
151    }
152
153    /**
154     * Read a File as XML, and return the root object.
155     * <p>
156     * Exceptions are only thrown when local recovery is impossible.
157     *
158     * @param file File to be parsed. A FileNotFoundException is thrown if it
159     *             doesn't exist.
160     * @throws org.jdom2.JDOMException       only when all methods have failed
161     * @throws java.io.FileNotFoundException if file not found
162     * @return root element from the file. This should never be null, as an
163     *         exception should be thrown if anything goes wrong.
164     */
165    public Element rootFromFile(File file) throws JDOMException, IOException {
166        if (log.isDebugEnabled()) {
167            log.debug("reading xml from file: {}", file.getPath());
168        }
169
170        try (FileInputStream fs = new FileInputStream(file)) {
171            return getRoot(fs);
172        }
173    }
174
175    /**
176     * Read an {@link java.io.InputStream} as XML, and return the root object.
177     * <p>
178     * Exceptions are only thrown when local recovery is impossible.
179     *
180     * @param stream InputStream to be parsed.
181     * @throws org.jdom2.JDOMException       only when all methods have failed
182     * @throws java.io.FileNotFoundException if file not found
183     * @return root element from the file. This should never be null, as an
184     *         exception should be thrown if anything goes wrong.
185     */
186    public Element rootFromInputStream(InputStream stream) throws JDOMException, IOException {
187        return getRoot(stream);
188    }
189
190    /**
191     * Read a URL as XML, and return the root object.
192     * <p>
193     * Exceptions are only thrown when local recovery is impossible.
194     *
195     * @param url URL locating the data file
196     * @throws org.jdom2.JDOMException only when all methods have failed
197     * @throws FileNotFoundException   if file not found
198     * @return root element from the file. This should never be null, as an
199     *         exception should be thrown if anything goes wrong.
200     */
201    public Element rootFromURL(URL url) throws JDOMException, IOException {
202        if (log.isDebugEnabled()) {
203            log.debug("reading xml from URL: {}", url.toString());
204        }
205        return getRoot(url.openConnection().getInputStream());
206    }
207
208    /**
209     * Get the root element from an XML document in a stream.
210     *
211     * @param stream input containing the XML document
212     * @return the root element of the XML document
213     * @throws org.jdom2.JDOMException if the XML document is invalid
214     * @throws java.io.IOException     if the input cannot be read
215     */
216    protected Element getRoot(InputStream stream) throws JDOMException, IOException {
217        log.trace("getRoot from stream");
218
219        processingInstructionHRef = null;
220        processingInstructionType = null;
221
222        SAXBuilder builder = getBuilder(getValidate());
223        Document doc = builder.build(new BufferedInputStream(stream));
224        doc = processInstructions(doc);  // handle any process instructions
225        // find root
226        return doc.getRootElement();
227    }
228
229    /**
230     * Write a File as XML.
231     *
232     * @param file File to be created.
233     * @param doc  Document to be written out. This should never be null.
234     * @throws IOException when an IO error occurs
235     * @throws FileNotFoundException if file not found
236     */
237    public void writeXML(File file, Document doc) throws IOException, FileNotFoundException {
238        // ensure parent directory exists
239        if (file.getParent() != null) {
240            FileUtil.createDirectory(file.getParent());
241        }
242        // write the result to selected file
243        try (FileOutputStream o = new FileOutputStream(file)) {
244            XMLOutputter fmt = new XMLOutputter();
245            fmt.setFormat(Format.getPrettyFormat()
246                    .setLineSeparator(System.getProperty("line.separator"))
247                    .setTextMode(Format.TextMode.TRIM_FULL_WHITE));
248            fmt.output(doc, o);
249            o.flush();
250        }
251    }
252
253    /**
254     * Check if a file of the given name exists. This uses the same search order
255     * as {@link #findFile}
256     *
257     * @param name file name, either absolute or relative
258     * @return true if the file exists in a searched place
259     */
260    protected boolean checkFile(String name) {
261        File fp = new File(name);
262        if (fp.exists()) {
263            return true;
264        }
265        fp = new File(FileUtil.getUserFilesPath() + name);
266        if (fp.exists()) {
267            return true;
268        } else {
269            File fx = new File(xmlDir() + name);
270            return fx.exists();
271        }
272    }
273
274    /**
275     * Get a File object for a name. This is here to implement the search
276     * rule:
277     * <ol>
278     *   <li>Look in user preferences directory, located by {@link jmri.util.FileUtil#getUserFilesPath()}
279     *   <li>Look in current working directory (usually the JMRI distribution directory)
280     *   <li>Look in program directory, located by {@link jmri.util.FileUtil#getProgramPath()}
281     *   <li>Look in XML directory, located by {@link #xmlDir}
282     *   <li>Check for absolute name.
283     * </ol>
284     *
285     * @param name Filename perhaps containing subdirectory information (e.g.
286     *             "decoders/Mine.xml")
287     * @return null if file found, otherwise the located File
288     */
289    protected File findFile(String name) {
290        URL url = FileUtil.findURL(name,
291                FileUtil.getUserFilesPath(),
292                ".",
293                FileUtil.getProgramPath(),
294                xmlDir());
295        if (url != null) {
296            try {
297                return new File(url.toURI());
298            } catch (URISyntaxException ex) {
299                return null;
300            }
301        }
302        return null;
303    }
304
305    /**
306     * Diagnostic printout of as much as we can find
307     *
308     * @param name Element to print, should not be null
309     */
310    static public void dumpElement(@Nonnull Element name) {
311        name.getChildren().forEach((element) -> {
312            log.info(" Element: {} ns: {}", element.getName(), element.getNamespace());
313        });
314    }
315
316    /**
317     * Move original file to a backup. Use this before writing out a new version
318     * of the file.
319     *
320     * @param name Last part of file pathname i.e. subdir/name, without the
321     *             pathname for either the xml or preferences directory.
322     */
323    public void makeBackupFile(String name) {
324        File file = findFile(name);
325        if (file == null) {
326            log.info("No {} file to backup", name);
327        } else if (file.canWrite()) {
328            String backupName = backupFileName(file.getAbsolutePath());
329            File backupFile = findFile(backupName);
330            if (backupFile != null) {
331                if (backupFile.delete()) {
332                    log.debug("deleted backup file {}", backupName);
333                }
334            }
335            if (file.renameTo(new File(backupName))) {
336                log.debug("created new backup file {}", backupName);
337            } else {
338                log.error("could not create backup file {}", backupName);
339            }
340        }
341    }
342
343    /**
344     * Move original file to backup directory.
345     *
346     * @param directory the backup directory to use.
347     * @param file      the file to be backed up. The file name will have the
348     *                  current date embedded in the backup name.
349     * @return true if successful.
350     */
351    public boolean makeBackupFile(String directory, File file) {
352        if (file == null) {
353            log.info("No file to backup");
354        } else if (file.canWrite()) {
355            String backupFullName = directory + File.separator + createFileNameWithDate(file.getName());
356            if (log.isDebugEnabled()) {
357                log.debug("new backup file: {}", backupFullName);
358            }
359
360            File backupFile = findFile(backupFullName);
361            if (backupFile != null) {
362                if (backupFile.delete()) {
363                    if (log.isDebugEnabled()) {
364                        log.debug("deleted backup file {}", backupFullName);
365                    }
366                }
367            } else {
368                backupFile = new File(backupFullName);
369            }
370            // create directory if needed
371            File parentDir = backupFile.getParentFile();
372            if (!parentDir.exists()) {
373                if (log.isDebugEnabled()) {
374                    log.debug("creating backup directory: {}", parentDir.getName());
375                }
376                if (!parentDir.mkdirs()) {
377                    log.error("backup directory not created");
378                    return false;
379                }
380            }
381            if (file.renameTo(new File(backupFullName))) {
382                if (log.isDebugEnabled()) {
383                    log.debug("created new backup file {}", backupFullName);
384                }
385            } else {
386                if (log.isDebugEnabled()) {
387                    log.debug("could not create backup file {}", backupFullName);
388                }
389                return false;
390            }
391        }
392        return true;
393    }
394
395    /**
396     * Revert to original file from backup. Use this for testing backup files.
397     *
398     * @param name Last part of file pathname i.e. subdir/name, without the
399     *             pathname for either the xml or preferences directory.
400     */
401    public void revertBackupFile(String name) {
402        File file = findFile(name);
403        if (file == null) {
404            log.info("No {} file to revert", name);
405        } else {
406            String backupName = backupFileName(file.getAbsolutePath());
407            File backupFile = findFile(backupName);
408            if (backupFile != null) {
409                log.info("No {} backup file to revert", backupName);
410                if (file.delete()) {
411                    log.debug("deleted original file {}", name);
412                }
413
414                if (backupFile.renameTo(new File(name))) {
415                    log.debug("created original file {}", name);
416                } else {
417                    log.error("could not create original file {}", name);
418                }
419            }
420        }
421    }
422
423    /**
424     * Return the name of a new, unique backup file. This is here so it can be
425     * overridden during tests. File to be backed-up must be within the
426     * preferences directory tree.
427     *
428     * @param name Filename without preference path information, e.g.
429     *             "decoders/Mine.xml".
430     * @return Complete filename, including path information into preferences
431     *         directory
432     */
433    public String backupFileName(String name) {
434        String f = name + ".bak";
435        if (log.isDebugEnabled()) {
436            log.debug("backup file name is: {}", f);
437        }
438        return f;
439    }
440
441    public String createFileNameWithDate(String name) {
442        // remove .xml extension
443        String[] fileName = name.split(".xml");
444        String f = fileName[0] + "_" + getDate() + ".xml";
445        if (log.isDebugEnabled()) {
446            log.debug("backup file name is: {}", f);
447        }
448        return f;
449    }
450
451    /**
452     * @return String based on the current date in the format of year month day
453     *         hour minute second. The date is fixed length and always returns a
454     *         date represented by 14 characters.
455     */
456    private String getDate() {
457        Calendar now = Calendar.getInstance();
458        return String.format("%d%02d%02d%02d%02d%02d",
459                now.get(Calendar.YEAR),
460                now.get(Calendar.MONTH) + 1,
461                now.get(Calendar.DATE),
462                now.get(Calendar.HOUR_OF_DAY),
463                now.get(Calendar.MINUTE),
464                now.get(Calendar.SECOND)
465        );
466    }
467
468    /**
469     * Execute the Processing Instructions in the file.
470     * <p>
471     * JMRI only knows about certain ones; the others will be ignored.
472     *
473     * @param doc the document containing processing instructions
474     * @return the processed document
475     */
476    Document processInstructions(Document doc) {
477        // this iterates over top level
478        for (Content c : doc.cloneContent()) {
479            if (c instanceof ProcessingInstruction) {
480                ProcessingInstruction pi = (ProcessingInstruction) c;
481                for (String attrName : pi.getPseudoAttributeNames()) {
482                    if ("href".equals(attrName)) {
483                        processingInstructionHRef = pi.getPseudoAttributeValue(attrName);
484                    }
485                    if ("type".equals(attrName)) {
486                        processingInstructionType = pi.getPseudoAttributeValue(attrName);
487                    }
488                }
489                try {
490                    doc = processOneInstruction((ProcessingInstruction) c, doc);
491                } catch (org.jdom2.transform.XSLTransformException ex) {
492                    log.error("XSLT error while transforming with {}, ignoring transform", c, ex);
493                } catch (org.jdom2.JDOMException ex) {
494                    log.error("JDOM error while transforming with {}, ignoring transform", c, ex);
495                } catch (java.io.IOException ex) {
496                    log.error("IO error while transforming with {}, ignoring transform", c, ex);
497                }
498            }
499        }
500
501        return doc;
502    }
503
504    Document processOneInstruction(ProcessingInstruction p, Document doc) throws org.jdom2.transform.XSLTransformException, org.jdom2.JDOMException, java.io.IOException {
505        log.trace("handling {}", p);
506
507        // check target
508        String target = p.getTarget();
509        if (!target.equals("transform-xslt")) {
510            return doc;
511        }
512
513        String href = p.getPseudoAttributeValue("href");
514        // we expect this to start with http://jmri.org/ and refer to the JMRI file tree
515        if (!href.startsWith("http://jmri.org/")) {
516            return doc;
517        }
518        href = href.substring(16);
519
520        // if starts with 'xml/' we remove that; findFile will put it back
521        if (href.startsWith("xml/")) {
522            href = href.substring(4);
523        }
524
525        // read the XSLT transform into a Document to get XInclude done
526        SAXBuilder builder = getBuilder(Validate.None);
527        Document xdoc = builder.build(new BufferedInputStream(new FileInputStream(findFile(href))));
528        org.jdom2.transform.XSLTransformer transformer = new org.jdom2.transform.XSLTransformer(xdoc);
529        return transformer.transform(doc);
530    }
531
532    /**
533     * Create the Document object to store a particular root Element.
534     *
535     * @param root Root element of the final document
536     * @param dtd  name of an external DTD
537     * @return new Document, with root installed
538     */
539    static public Document newDocument(Element root, String dtd) {
540        Document doc = new Document(root);
541        doc.setDocType(new DocType(root.getName(), dtd));
542        addDefaultInfo(root);
543        return doc;
544    }
545
546    /**
547     * Create the Document object to store a particular root Element, without a
548     * DocType DTD (e.g. for using a schema)
549     *
550     * @param root Root element of the final document
551     * @return new Document, with root installed
552     */
553    static public Document newDocument(Element root) {
554        Document doc = new Document(root);
555        addDefaultInfo(root);
556        return doc;
557    }
558
559    /**
560     * Add default information to the XML before writing it out.
561     * <p>
562     * Currently, this is identification information as an XML comment. This
563     * includes: <ul>
564     * <li>The JMRI version used <li>Date of writing <li>A CVS id string, in
565     * case the file gets checked in or out </ul>
566     * <p>
567     * It may be necessary to extend this to check whether the info is already
568     * present, e.g. if re-writing a file.
569     *
570     * @param root The root element of the document that will be written.
571     */
572    static public void addDefaultInfo(Element root) {
573        var loadAndStorePreferences = InstanceManager.getDefault(LoadAndStorePreferences.class);
574        if (!loadAndStorePreferences.isExcludeJmriVersion()) {
575            String content = "Written by JMRI version " + jmri.Version.name()
576                + " on " + (new Date()).toString();
577            Comment comment = new Comment(content);
578            root.addContent(comment);
579        }
580    }
581
582    /**
583     * Define the location of XML files within the distribution directory.
584     * <p>
585     * Use {@link FileUtil#getProgramPath()} since the current working directory
586     * is not guaranteed to be the JMRI distribution directory if jmri.jar is
587     * referenced by an external Java application.
588     *
589     * @return the XML directory that ships with JMRI.
590     */
591    static public String xmlDir() {
592        return FileUtil.getProgramPath() + "xml" + File.separator;
593    }
594
595    /**
596     * Whether to, by global default, validate the file being read. Public so it
597     * can be set by scripting and for debugging.
598     *
599     * @return the default level of validation to apply to a file
600     */
601    static public Validate getDefaultValidate() {
602        return defaultValidate;
603    }
604
605    static public void setDefaultValidate(Validate v) {
606        defaultValidate = v;
607    }
608
609    static private Validate defaultValidate = Validate.None;
610
611    /**
612     * Whether to verify the DTD of this XML file when read.
613     *
614     * @return the level of validation to apply to a file
615     */
616    public Validate getValidate() {
617        return validate;
618    }
619
620    public void setValidate(Validate v) {
621        validate = v;
622    }
623
624    private Validate validate = defaultValidate;
625
626    /**
627     * Get the default standard location for DTDs in new XML documents. Public
628     * so it can be set by scripting and for debug.
629     *
630     * @return the default DTD location
631     */
632    static public String getDefaultDtdLocation() {
633        return defaultDtdLocation;
634    }
635
636    static public void setDefaultDtdLocation(String v) {
637        defaultDtdLocation = v;
638    }
639
640    static String defaultDtdLocation = "/xml/DTD/";
641
642    /**
643     * Get the location for DTDs in this XML document.
644     *
645     * @return the DTD location
646     */
647    public String getDtdLocation() {
648        return dtdLocation;
649    }
650
651    public void setDtdLocation(String v) {
652        dtdLocation = v;
653    }
654
655    public String dtdLocation = defaultDtdLocation;
656
657    /**
658     * Provide a JFileChooser initialized to the default user location, and with
659     * a default filter. This filter excludes {@code .zip} and {@code .jar}
660     * archives.
661     *
662     * @param filter Title for the filter, may not be null
663     * @param suffix Allowed file extensions, if empty all extensions are
664     *               allowed except {@code .zip} and {@code .jar}; include an
665     *               empty String to allow files without an extension if
666     *               specifying other extensions.
667     * @return a file chooser
668     */
669    public static JFileChooser userFileChooser(String filter, String... suffix) {
670        JFileChooser fc = new jmri.util.swing.JmriJFileChooser(FileUtil.getUserFilesPath());
671        fc.setFileFilter(new NoArchiveFileFilter(filter, suffix));
672        return fc;
673    }
674
675    /**
676     * Provide a JFileChooser initialized to the default user location, and with
677     * a default filter. This filter excludes {@code .zip} and {@code .jar}
678     * archives.
679     *
680     * @return a file chooser
681     */
682    public static JFileChooser userFileChooser() {
683        JFileChooser fc = new jmri.util.swing.JmriJFileChooser(FileUtil.getUserFilesPath());
684        fc.setFileFilter(new NoArchiveFileFilter());
685        return fc;
686    }
687
688    @SuppressWarnings("deprecation") // org.jdom2.input.SAXBuilder(java.lang.String saxDriverClass, boolean validate)
689    //{@see http://www.jdom.org/docs/apidocs/org/jdom2/input/SAXBuilder.html}
690    //{@see http://www.jdom.org/docs/apidocs/org/jdom2/input/sax/XMLReaders.html#NONVALIDATING}
691    // Validate.CheckDtdThenSchema may not be available readily
692    public static SAXBuilder getBuilder(Validate validate) {  // should really be a Verify enum
693        SAXBuilder builder;
694
695        boolean verifyDTD = (validate == Validate.CheckDtd) || (validate == Validate.CheckDtdThenSchema);
696        boolean verifySchema = (validate == Validate.RequireSchema) || (validate == Validate.CheckDtdThenSchema);
697
698        // old style
699        builder = new SAXBuilder("org.apache.xerces.parsers.SAXParser", verifyDTD);  // argument controls DTD validation
700
701        // insert local resolver for includes, schema, DTDs
702        builder.setEntityResolver(new JmriLocalEntityResolver());
703
704        // configure XInclude handling
705        builder.setFeature("http://apache.org/xml/features/xinclude", true);
706        builder.setFeature("http://apache.org/xml/features/xinclude/fixup-base-uris", false);
707
708        // only validate if grammar is available, making ABSENT OK
709        builder.setFeature("http://apache.org/xml/features/validation/dynamic", verifyDTD && !verifySchema);
710
711        // control Schema validation
712        builder.setFeature("http://apache.org/xml/features/validation/schema", verifySchema);
713        builder.setFeature("http://apache.org/xml/features/validation/schema-full-checking", verifySchema);
714
715        // if not validating DTD, just validate Schema
716        builder.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", verifyDTD);
717        if (!verifyDTD) {
718            builder.setProperty("http://java.sun.com/xml/jaxp/properties/schemaLanguage", "http://www.w3.org/2001/XMLSchema");
719        }
720
721        // allow Java character encodings
722        builder.setFeature("http://apache.org/xml/features/allow-java-encodings", true);
723
724        return builder;
725    }
726
727    // initialize logging
728    private static final Logger log = LoggerFactory.getLogger(XmlFile.class);
729
730}