Added configurable robots.txt file.

Deleted old static file.
Default output depends on app-public setting.
Otherwise can be overidden in `.env` file via `ALLOW_ROBOTS`
Otherwise view file can be customized.

Fixes #779

diff --git a/app/Http/Controllers/HomeController.php b/app/Http/Controllers/HomeController.php
index 3a5fd2cb5c53ccba1a336ca6c05b0a8cfb2cd4dd..bbe1a86799bc365b107f80c3059fac7a7dafb0ef 100644 (file)
--- a/app/Http/Controllers/HomeController.php
+++ b/app/Http/Controllers/HomeController.php
@@ -118,4 +118,20 @@ class HomeController extends Controller
     {
         return view('partials/custom-head-content');
     }
+
+    /**
+     * Show the view for /robots.txt
+     * @return $this
+     */
+    public function getRobots()
+    {
+        $sitePublic = setting('app-public', false);
+        $allowRobots = config('app.allow_robots');
+        if ($allowRobots === null) {
+            $allowRobots = $sitePublic;
+        }
+        return response()
+            ->view('robots', ['allowRobots' => $allowRobots])
+            ->header('Content-Type', 'text/plain');
+    }
 }

diff --git a/config/app.php b/config/app.php
index fad0c20f247fb97c84833f268c3c545c55ef6465..ca30ee2a0778b0ec1a61e0ab8f96d2e5a6a9d08c 100755 (executable)
--- a/config/app.php
+++ b/config/app.php
@@ -4,12 +4,28 @@ return [
 
     'env' => env('APP_ENV', 'production'),
 
+    /**
+     * Set the default view type for various lists. Can be overridden by user preferences.
+     * This will be used for public viewers and users that have not set a preference.
+     */
     'views' => [
         'books' => env('APP_VIEWS_BOOKS', 'list')
     ],
 
+    /**
+     * Allow <script> tags to entered within page content.
+     * <script> tags are escaped by default.
+     * Even when overridden the WYSIWYG editor may still escape script content.
+     */
     'allow_content_scripts' => env('ALLOW_CONTENT_SCRIPTS', false),
 
+    /**
+     * Override the default behaviour for allowing crawlers to crawl the instance.
+     * May be ignored if view has be overridden or modified.
+     * Defaults to null since, if not set, 'app-public' status used instead.
+     */
+    'allow_robots' => env('ALLOW_ROBOTS', null),
+
     /*
     |--------------------------------------------------------------------------
     | Application Debug Mode

diff --git a/public/robots.txt b/public/robots.txt
deleted file mode 100644 (file)
index eb05362..0000000
--- a/public/robots.txt
+++ /dev/null
@@ -1,2 +0,0 @@
-User-agent: *
-Disallow:

diff --git a/resources/views/robots.blade.php b/resources/views/robots.blade.php
new file mode 100644 (file)
index 0000000..f3d1079
--- /dev/null
+++ b/resources/views/robots.blade.php
@@ -0,0 +1,6 @@
+User-agent: *
+@if($allowRobots)
+Disallow:
+@else
+Disallow: /
+@endif
\ No newline at end of file

diff --git a/routes/web.php b/routes/web.php
index 7a8634ca39ec659e37142aa47df38f58cb25a6ab..be05cda90ad87a50cf14dc74e75568e92ae52b92 100644 (file)
--- a/routes/web.php
+++ b/routes/web.php
@@ -2,6 +2,7 @@
 
 Route::get('/translations', 'HomeController@getTranslations');
 Route::get('/icon/{iconName}.svg', 'HomeController@getIcon');
+Route::get('/robots.txt', 'HomeController@getRobots');
 
 // Authenticated routes...
 Route::group(['middleware' => 'auth'], function () {

diff --git a/tests/PublicActionTest.php b/tests/PublicActionTest.php
index 6f8590d4b7b19f8a52d76381f2438527ac0e4b32..dadb37e46ce2cac9082d98b05fef2615fee972db 100644 (file)
--- a/tests/PublicActionTest.php
+++ b/tests/PublicActionTest.php
@@ -90,4 +90,35 @@ class PublicActionTest extends BrowserKitTest
         $this->dontSee($page->name);
     }
 
+    public function test_robots_effected_by_public_status()
+    {
+        $this->visit('/robots.txt');
+        $this->seeText("User-agent: *\nDisallow: /");
+
+        $this->setSettings(['app-public' => 'true']);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow:");
+        $this->dontSeeText("Disallow: /");
+    }
+
+    public function test_robots_effected_by_setting()
+    {
+        $this->visit('/robots.txt');
+        $this->seeText("User-agent: *\nDisallow: /");
+
+        config()->set('app.allow_robots', true);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow:");
+        $this->dontSeeText("Disallow: /");
+
+        // Check config overrides app-public setting
+        config()->set('app.allow_robots', false);
+        $this->setSettings(['app-public' => 'true']);
+        $this->visit('/robots.txt');
+
+        $this->seeText("User-agent: *\nDisallow: /");
+    }
+
 }
\ No newline at end of file