robots.txt + periodically xml sitemaps generation.

.gitignore

@@ -2,6 +2,8 @@
 *.swp
 nimcache/
 *.db*
+public/sitemap*.xml
+public/robots.txt
 
 # Specific paths
 /createdb

nimforum.nimble

@@ -1,5 +1,5 @@
 # Package
-version       = "2.2.0"
+version       = "2.3.0"
 author        = "Dominik Picheta"
 description   = "The Nim forum"
 license       = "MIT"
@@ -23,7 +23,7 @@ requires "sass#649e0701fa5c"
 requires "karax#45bac6b"
 
 requires "webdriver#c5e4182"
-
+requires "https://github.com/enthus1ast/nimSimpleSitemap.git#f62730a"
 when NimMajor >= 1 and NimMinor >= 9:
   requires "db_connector >= 0.1.0"
   requires "smtp >= 0.1.0"

src/forum.nim

@@ -14,7 +14,7 @@ import
 import cgi except setCookie
 import std/options
 
-import auth, email, utils, buildcss
+import auth, email, utils, buildcss, sitemap
 
 when NimMajor >= 1 and NimMinor >= 9:
   import db_connector/db_sqlite
@@ -50,6 +50,7 @@ var
   config: Config
   captcha: ReCaptcha
   mailer: Mailer
+  sitemapGenerator: SitemapGenerator
   karaxHtml: string
 
 proc init(c: TForumData) =
@@ -253,6 +254,13 @@ proc verifyIdentHash(
   if newIdent != ident:
     raise newForumError("Invalid ident hash")
 
+proc periodicallyCreateSitemap() {.async.} =
+  while true:
+    info "Regenerate sitemap"
+    sitemapGenerator.generate()
+    await sleepAsync(1000 * 60 * 60 * 24) # One day
+
+
 proc initialise() =
   randomize()
 
@@ -270,6 +278,10 @@ proc initialise() =
   isFTSAvailable = db.getAllRows(sql("SELECT name FROM sqlite_master WHERE " &
       "type='table' AND name='post_fts'")).len == 1
 
+  sitemapGenerator = newSitemapGenerator(db, config.hostname)
+  info "Generate sitemap on start"
+  sitemapGenerator.generate()
+  asyncCheck periodicallyCreateSitemap()
   buildCSS(config)
 
   # Read karax.html and set its properties.
@@ -802,7 +814,7 @@ proc updateProfile(
   if c.rank < Moderator and wasEmailChanged:
     if rank != EmailUnconfirmed:
       raise newForumError("Rank needs a change when setting new email.")
-    
+
     {.cast(gcsafe).}:
       await sendSecureEmail(
         mailer, ActivateEmail, c.req, row[0], row[1], email, row[3]

src/setup_nimforum.nim

@@ -7,7 +7,7 @@
 #
 # Script to initialise the nimforum.
 
-import strutils, os, times, json, options, terminal
+import strutils, os, times, json, options, terminal, uri
 
 when NimMajor >= 1 and NimMinor >= 9:
   import db_connector/db_sqlite
@@ -320,6 +320,9 @@ These can be changed later in the generated forum.json file.
     dbPath
   )
 
+  echo("Creating public/robots.txt")
+  writeFile("public" / "robots.txt", "Sitemap: " & $(parseUri(hostname) / "sitemap.xml"))
+
   echo("Setup complete!")
 
 proc echoHelp() =

src/sitemap.nim

@@ -0,0 +1,33 @@
+import simpleSitemap, times, os, uri
+
+when NimMajor >= 1 and NimMinor >= 9:
+  import db_connector/db_sqlite
+else:
+  import std/db_sqlite
+
+type
+  SitemapGenerator* = object
+    dbconn: Dbconn
+    hostname: Uri
+
+proc newSitemapGenerator*(dbconn: DbConn, hostname: string): SitemapGenerator =
+  ## The generator for sitemaps `dbconn` is the forums database, `hostname` is the absolute url
+  ## the forum runs on.
+  result = SitemapGenerator()
+  result.dbconn = dbconn
+  result.hostname = parseUri(hostname)
+
+proc generate*(sg: SitemapGenerator) =
+  ## Generates the xml sitemap(s) creates them in `getAppDir() / "public"`
+  var urlDates: seq[UrlDate] = @[]
+  for row in sg.dbconn.rows(sql"select id, modified from thread"):
+    let absurl = sg.hostname / "t" / row[0]
+    let pdate = row[1].parse("yyyy-MM-dd H':'m':'s")
+    urlDates.add ($absurl, pdate)
+  let pages = generateSitemaps(
+    urlDates,
+    urlsOnRecent = 30,
+    maxUrlsPerSitemap = 50_000,
+    base = $sg.hostname
+  )
+  write(pages, folder = getAppDir() / "public")