diff --git a/.gitignore b/.gitignore index fe26a8a..fe013d9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ *.swp nimcache/ *.db* +public/sitemap*.xml +public/robots.txt # Specific paths /createdb diff --git a/nimforum.nimble b/nimforum.nimble index 68eb8c0..11f96cc 100644 --- a/nimforum.nimble +++ b/nimforum.nimble @@ -1,5 +1,5 @@ # Package -version = "2.2.0" +version = "2.3.0" author = "Dominik Picheta" description = "The Nim forum" license = "MIT" @@ -23,7 +23,7 @@ requires "sass#649e0701fa5c" requires "karax#45bac6b" requires "webdriver#c5e4182" - +requires "https://github.com/enthus1ast/nimSimpleSitemap.git#f62730a" when NimMajor >= 1 and NimMinor >= 9: requires "db_connector >= 0.1.0" requires "smtp >= 0.1.0" diff --git a/src/forum.nim b/src/forum.nim index 1da9616..d477fc2 100644 --- a/src/forum.nim +++ b/src/forum.nim @@ -14,7 +14,7 @@ import import cgi except setCookie import std/options -import auth, email, utils, buildcss +import auth, email, utils, buildcss, sitemap when NimMajor >= 1 and NimMinor >= 9: import db_connector/db_sqlite @@ -50,6 +50,7 @@ var config: Config captcha: ReCaptcha mailer: Mailer + sitemapGenerator: SitemapGenerator karaxHtml: string proc init(c: TForumData) = @@ -253,6 +254,13 @@ proc verifyIdentHash( if newIdent != ident: raise newForumError("Invalid ident hash") +proc periodicallyCreateSitemap() {.async.} = + while true: + info "Regenerate sitemap" + sitemapGenerator.generate() + await sleepAsync(1000 * 60 * 60 * 24) # One day + + proc initialise() = randomize() @@ -270,6 +278,10 @@ proc initialise() = isFTSAvailable = db.getAllRows(sql("SELECT name FROM sqlite_master WHERE " & "type='table' AND name='post_fts'")).len == 1 + sitemapGenerator = newSitemapGenerator(db, config.hostname) + info "Generate sitemap on start" + sitemapGenerator.generate() + asyncCheck periodicallyCreateSitemap() buildCSS(config) # Read karax.html and set its properties. @@ -802,7 +814,7 @@ proc updateProfile( if c.rank < Moderator and wasEmailChanged: if rank != EmailUnconfirmed: raise newForumError("Rank needs a change when setting new email.") - + {.cast(gcsafe).}: await sendSecureEmail( mailer, ActivateEmail, c.req, row[0], row[1], email, row[3] diff --git a/src/setup_nimforum.nim b/src/setup_nimforum.nim index 0485b7f..3630541 100644 --- a/src/setup_nimforum.nim +++ b/src/setup_nimforum.nim @@ -7,7 +7,7 @@ # # Script to initialise the nimforum. -import strutils, os, times, json, options, terminal +import strutils, os, times, json, options, terminal, uri when NimMajor >= 1 and NimMinor >= 9: import db_connector/db_sqlite @@ -320,6 +320,9 @@ These can be changed later in the generated forum.json file. dbPath ) + echo("Creating public/robots.txt") + writeFile("public" / "robots.txt", "Sitemap: " & $(parseUri(hostname) / "sitemap.xml")) + echo("Setup complete!") proc echoHelp() = diff --git a/src/sitemap.nim b/src/sitemap.nim new file mode 100644 index 0000000..996f358 --- /dev/null +++ b/src/sitemap.nim @@ -0,0 +1,33 @@ +import simpleSitemap, times, os, uri + +when NimMajor >= 1 and NimMinor >= 9: + import db_connector/db_sqlite +else: + import std/db_sqlite + +type + SitemapGenerator* = object + dbconn: Dbconn + hostname: Uri + +proc newSitemapGenerator*(dbconn: DbConn, hostname: string): SitemapGenerator = + ## The generator for sitemaps `dbconn` is the forums database, `hostname` is the absolute url + ## the forum runs on. + result = SitemapGenerator() + result.dbconn = dbconn + result.hostname = parseUri(hostname) + +proc generate*(sg: SitemapGenerator) = + ## Generates the xml sitemap(s) creates them in `getAppDir() / "public"` + var urlDates: seq[UrlDate] = @[] + for row in sg.dbconn.rows(sql"select id, modified from thread"): + let absurl = sg.hostname / "t" / row[0] + let pdate = row[1].parse("yyyy-MM-dd H':'m':'s") + urlDates.add ($absurl, pdate) + let pages = generateSitemaps( + urlDates, + urlsOnRecent = 30, + maxUrlsPerSitemap = 50_000, + base = $sg.hostname + ) + write(pages, folder = getAppDir() / "public") \ No newline at end of file