From: Dan Brown <redacted>
Date: Sat, 31 Mar 2018 11:41:40 +0000 (+0100)
Subject: Added configurable robots.txt file.
X-Git-Tag: v0.21.0~1^2~18
X-Git-Url: http://source.bookstackapp.com/bookstack/commitdiff_plain/1a72208d27e1b27034fd9eb5f657d6f3cd0c5ad7?ds=sidebyside;hp=--cc

Added configurable robots.txt file.

Deleted old static file.
Default output depends on app-public setting.
Otherwise can be overidden in `.env` file via `ALLOW_ROBOTS`
Otherwise view file can be customized.

Fixes #779
---

1a72208d27e1b27034fd9eb5f657d6f3cd0c5ad7
diff --git a/app/Http/Controllers/HomeController.php b/app/Http/Controllers/HomeController.php
index 3a5fd2cb5..bbe1a8679 100644
--- a/app/Http/Controllers/HomeController.php
+++ b/app/Http/Controllers/HomeController.php
@@ -118,4 +118,20 @@ class HomeController extends Controller
     {
         return view('partials/custom-head-content');
     }
+
+    /**
+     * Show the view for /robots.txt
+     * @return $this
+     */
+    public function getRobots()
+    {
+        $sitePublic = setting('app-public', false);
+        $allowRobots = config('app.allow_robots');
+        if ($allowRobots === null) {
+            $allowRobots = $sitePublic;
+        }
+        return response()
+            ->view('robots', ['allowRobots' => $allowRobots])
+            ->header('Content-Type', 'text/plain');
+    }
 }
diff --git a/config/app.php b/config/app.php
index fad0c20f2..ca30ee2a0 100755
--- a/config/app.php
+++ b/config/app.php
@@ -4,12 +4,28 @@ return [
 
     'env' => env('APP_ENV', 'production'),
 
+    /**
+     * Set the default view type for various lists. Can be overridden by user preferences.
+     * This will be used for public viewers and users that have not set a preference.
+     */
     'views' => [
         'books' => env('APP_VIEWS_BOOKS', 'list')
     ],
 
+    /**
+     * Allow <script> tags to entered within page content.
+     * <script> tags are escaped by default.
+     * Even when overridden the WYSIWYG editor may still escape script content.
+     */
     'allow_content_scripts' => env('ALLOW_CONTENT_SCRIPTS', false),
 
+    /**
+     * Override the default behaviour for allowing crawlers to crawl the instance.
+     * May be ignored if view has be overridden or modified.
+     * Defaults to null since, if not set, 'app-public' status used instead.
+     */
+    'allow_robots' => env('ALLOW_ROBOTS', null),
+
     /*
     |--------------------------------------------------------------------------
     | Application Debug Mode
diff --git a/public/robots.txt b/public/robots.txt
deleted file mode 100644
index eb0536286..000000000
--- a/public/robots.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-User-agent: *
-Disallow:
diff --git a/resources/views/robots.blade.php b/resources/views/robots.blade.php
new file mode 100644
index 000000000..f3d10797a
--- /dev/null
+++ b/resources/views/robots.blade.php
@@ -0,0 +1,6 @@
+User-agent: *
+@if($allowRobots)
+Disallow:
+@else
+Disallow: /
+@endif
\ No newline at end of file
diff --git a/routes/web.php b/routes/web.php
index 7a8634ca3..be05cda90 100644
--- a/routes/web.php
+++ b/routes/web.php
@@ -2,6 +2,7 @@
 
 Route::get('/translations', 'HomeController@getTranslations');
 Route::get('/icon/{iconName}.svg', 'HomeController@getIcon');
+Route::get('/robots.txt', 'HomeController@getRobots');
 
 // Authenticated routes...
 Route::group(['middleware' => 'auth'], function () {
diff --git a/tests/PublicActionTest.php b/tests/PublicActionTest.php
index 6f8590d4b..dadb37e46 100644
--- a/tests/PublicActionTest.php
+++ b/tests/PublicActionTest.php
@@ -90,4 +90,35 @@ class PublicActionTest extends BrowserKitTest
         $this->dontSee($page->name);
     }
 
+    public function test_robots_effected_by_public_status()
+    {
+        $this->visit('/robots.txt');
+        $this->seeText("User-agent: *\nDisallow: /");
+
+        $this->setSettings(['app-public' => 'true']);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow:");
+        $this->dontSeeText("Disallow: /");
+    }
+
+    public function test_robots_effected_by_setting()
+    {
+        $this->visit('/robots.txt');
+        $this->seeText("User-agent: *\nDisallow: /");
+
+        config()->set('app.allow_robots', true);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow:");
+        $this->dontSeeText("Disallow: /");
+
+        // Check config overrides app-public setting
+        config()->set('app.allow_robots', false);
+        $this->setSettings(['app-public' => 'true']);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow: /");
+    }
+
 }
\ No newline at end of file