Serving sitemap.xml and robots.txt with Spring MVC

Question

What is the best way to server sitemap.xml and robots.txt with Spring MVC? I want server these files through Controller in cleanest way.

Answer 1

This post was very useful for me, and I believe this is an update to the original resolution.

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElements;
import javax.xml.bind.annotation.XmlRootElement;

@XmlRootElement (name = "urlset")
@XmlAccessorType(XmlAccessType.NONE)
public class XmlUrlSet {
  
  @XmlElements({@XmlElement(name = "url", type = XmlUrl.class)})
  private Collection<XmlUrl> xmlUrls = new ArrayList<XmlUrl>();

  public void addUrl(XmlUrl xmlUrl) {
      xmlUrls.add(xmlUrl);
  }

  public Collection<XmlUrl> getXmlUrls() {
      return xmlUrls;
  }
}

---

import javax.xml.bind.annotation.XmlAccessType;
import javax.xml.bind.annotation.XmlAccessorType;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlRootElement;

@XmlRootElement (name = "url")
@XmlAccessorType(XmlAccessType.NONE)
public class XmlUrl {

    @XmlElement
    private String loc;

    @XmlElement
    private String lastmod;

    public XmlUrl() {
    }

    public XmlUrl(String loc, String date) {
        this.loc = loc;
        this.lastmod = date;
    }

    public String getLoc() {
        return loc;
    }

    public String getLastmod() {
        return lastmod;
    }
}

on the same package (this is like this because, I am working on defining multiple namespaces, because I am working on a more complex sitemap):

@XmlSchema(
  namespace="http://www.sitemaps.org/schemas/sitemap/0.9",
  xmlns={
    @XmlNs(prefix = "", namespaceURI = "http://www.sitemaps.org/schemas/sitemap/0.9")
  },
  elementFormDefault=XmlNsForm.QUALIFIED
)

package com.visite.backoffice.domain.sitemap;

import javax.xml.bind.annotation.XmlNs;
import javax.xml.bind.annotation.XmlNsForm;
import javax.xml.bind.annotation.XmlSchema;

Then the REST controller:

@RestController
@RequestMapping(path = "/api/site", produces = "application/xml")
public class ConfigurationResource {

    private final Logger log = LoggerFactory.getLogger(ConfigurationResource.class);

    public ConfigurationResource() {
    }

    @GetMapping("/configurations/{siteId}/site-map") 
    public ResponseEntity<XmlUrlSet> getSiteConfigurations( @PathVariable String siteId ) {
        log.debug("REST request to get sitemap for site {}", siteId);

        XmlUrlSet result = new XmlUrlSet();
        for (int i = 0; i < 10; i++) {
            String fakeUrl = "http://www.algo.com/" + i;
            LocalDate tmp = LocalDate.now();
            result.addUrl(new XmlUrl(fakeUrl, tmp.toString()));
        }
        
        return ResponseEntity.ok(result);
    }
}

result:

<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
    <url>
        <loc>http://www.algo.com/0</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/1</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/2</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/3</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/4</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/5</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/6</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/7</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/8</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
    <url>
        <loc>http://www.algo.com/9</loc>
        <lastmod>2022-05-12</lastmod>
    </url>
</urlset>

Answer 2

Add this line to your dispatcher servlet xml file:

<mvc:resources mapping="/robots.txt" location="/WEB-INF/robots.txt" order="0"/> 

Put the robots.txt at WEB-INF/robots.txt. The file will be accessible by yoursite.com/robots.txt

Answer 3

I'm relying on JAXB to generate the sitemap.xml for me.

My controller looks something like the below, and I have some database tables to keep track of the links that I want to appear in the sitemap:-

SitemapController.java

@Controller
public class SitemapController {

    @RequestMapping(value = "/sitemap.xml", method = RequestMethod.GET)
    @ResponseBody
    public XmlUrlSet main() {
        XmlUrlSet xmlUrlSet = new XmlUrlSet();
        create(xmlUrlSet, "", XmlUrl.Priority.HIGH);
        create(xmlUrlSet, "/link-1", XmlUrl.Priority.HIGH);
        create(xmlUrlSet, "/link-2", XmlUrl.Priority.MEDIUM);

        // for loop to generate all the links by querying against database
        ...

        return xmlUrlSet;
    }

    private void create(XmlUrlSet xmlUrlSet, String link, XmlUrl.Priority priority) {
        xmlUrlSet.addUrl(new XmlUrl("http://www.mysite.com" + link, priority));
    }

}

XmlUrl.java

@XmlAccessorType(value = XmlAccessType.NONE)
@XmlRootElement(name = "url")
public class XmlUrl {
    public enum Priority {
        HIGH("1.0"), MEDIUM("0.5");

        private String value;

        Priority(String value) {
            this.value = value;
        }

        public String getValue() {
            return value;
        }
    }

    @XmlElement
    private String loc;

    @XmlElement
    private String lastmod = new DateTime().toString(DateTimeFormat.forPattern("yyyy-MM-dd"));

    @XmlElement
    private String changefreq = "daily";

    @XmlElement
    private String priority;

    public XmlUrl() {
    }

    public XmlUrl(String loc, Priority priority) {
        this.loc = loc;
        this.priority = priority.getValue();
    }

    public String getLoc() {
        return loc;
    }

    public String getPriority() {
        return priority;
    }

    public String getChangefreq() {
        return changefreq;
    }

    public String getLastmod() {
        return lastmod;
    }
}

XmlUrlSet.java

@XmlAccessorType(value = XmlAccessType.NONE)
@XmlRootElement(name = "urlset", namespace = "http://www.sitemaps.org/schemas/sitemap/0.9")
public class XmlUrlSet {

    @XmlElements({@XmlElement(name = "url", type = XmlUrl.class)})
    private Collection<XmlUrl> xmlUrls = new ArrayList<XmlUrl>();

    public void addUrl(XmlUrl xmlUrl) {
        xmlUrls.add(xmlUrl);
    }

    public Collection<XmlUrl> getXmlUrls() {
        return xmlUrls;
    }
}

For the robots.txt, it looks something like the below, and obviously, you will need to configure it based on your likings:-

RobotsController.java

@Controller
public class RobotsController {

    @RequestMapping(value = "/robots.txt", method = RequestMethod.GET)
    public String getRobots(HttpServletRequest request) {
        return (Arrays.asList("mysite.com", "www.mysite.com").contains(request.getServerName())) ?
                "robotsAllowed" : "robotsDisallowed";
    }
}