diff --git a/content.example/robots.txt b/content.example/robots.txt new file mode 100644 index 0000000..0ad279c --- /dev/null +++ b/content.example/robots.txt @@ -0,0 +1,2 @@ +User-Agent: * +Disallow: diff --git a/server.py b/server.py index 0af9a91..2d9454c 100755 --- a/server.py +++ b/server.py @@ -321,6 +321,10 @@ def sendPicture(path): def icon(): return flask.send_from_directory('static', 'defaultFavicon.ico') +@app.route('/robots.txt') +def robots(): + return flask.send_from_directory(app.config["CONTENT_DIR"], 'robots.txt') + @app.route("/sitemap.xml") def siteMap(): '''Return an XML-sitemap for SEO''' @@ -356,9 +360,11 @@ def siteMap(): # add /content/ to sitemap # content = filter(lambda x: x.startswith(IDENTIFIER_PREFIX), app.config.keys()) - for c in content: + uniqueContent = filter(lambda x: not x.endswith(CONFIG_POSTFIX), content) + for c in uniqueContent: idWithoutPrefix = c.lstrip(IDENTIFIER_PREFIX) - urls += [("/content?id={}".format(idWithoutPrefix), n[PARSED_TIME], PRIORITY_PRIMARY)] + urls += [("/content?id={}".format(idWithoutPrefix), app.config["START_TIME"], + PRIORITY_PRIMARY)] hostname = flask.request.headers.get("X-REAL-HOSTNAME") if not hostname: