diff --git a/config.lisp b/config.lisp index a943447..397c669 100644 --- a/config.lisp +++ b/config.lisp @@ -2,11 +2,18 @@ (in-package #:quicklisp-controller) -(eval-when (:compile-toplevel :load-toplevel :execute) - (ubiquitous:restore 'quicklisp-controller)) +(defun config-value (name) + (let* ((base "quicklisp-controller:config;value.txt") + (file (make-pathname :name name :defaults base))) + (when (probe-file file) + (with-open-file (stream file) + (read-line stream))))) -(defvar *report-to-email* (ubiquitous:value 'report-to-email) + +(defvar *report-to-email* + (config-value "report-to-email") "The email address to which reports are emailed.") -(defparameter githappy:*oauth2-token* (ubiquitous:value 'github-access-token)) +(defparameter githappy:*oauth2-token* + (config-value "githappy-token")) diff --git a/daily.screenrc b/daily.screenrc new file mode 100644 index 0000000..752272f --- /dev/null +++ b/daily.screenrc @@ -0,0 +1 @@ +screen sbcl --non-interactive --no-userinit --no-sysinit --load daily-script.lisp diff --git a/daily.sh b/daily.sh new file mode 100755 index 0000000..a587934 --- /dev/null +++ b/daily.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +cd `dirname $0` +PATH=$PATH:/usr/local/bin +screen -L -c daily.screenrc -dmLS daily-build diff --git a/debian-setup/debian-9-packages.txt b/debian-setup/debian-9-packages.txt new file mode 100644 index 0000000..d55119e --- /dev/null +++ b/debian-setup/debian-9-packages.txt @@ -0,0 +1,146 @@ +swig +libopenmpi-dev +libusb-dev +libpython2.7-dev +curl +cvs +screen +build-essential +git-core +subversion +mercurial +darcs +zlib1g-dev +emacs-nox +libssl-dev +libyaml-dev +libffi6 +libffi-dev +libsdl1.2debian +libsdl-image1.2 +libsdl-gfx1.2-dev +libsdl-mixer1.2-dev +libuv1-dev +primus-libs +libpython2.7 +libgtkglext1-dev +libcairo2 +libfreetype6-dev +libx11-dev +libgtk2.0-dev +libglu1-mesa +freeglut3-dev +libdrm-dev +libgbm-dev +libegl1-mesa-dev +libsdl2-2.0-0 +libhdf5-dev +libgfortran3 +libblas-dev +liblapack-dev +libfixposix-dev +libglib2.0-dev +libpango-1.0-0 +libgdk-pixbuf2.0-0 +libgtk2.0-0 +libglib2.0-0 +libgtk-3-0 +libsqlite3-0 +libdevil-dev +libenchant-dev +libev-dev +libfam-dev +libfcgi-dev +libfbclient2 +libfreeimage-dev +libfuse-dev +libsqlite3-dev +libgeoip-dev +libgeos-dev +libgit2-dev +libftgl2 +libglu1-mesa-dev +libgl1-mesa-dev +libglfw3-dev +libglfw3 +libgirepository-1.0-1 +libgraphviz-dev +libkrb5-dev +libkyotocabinet-dev +liballegro-acodec5-dev +liballegro-audio5-dev +liballegro-dialog5-dev +liballegro-image5-dev +liballegro-physfs5-dev +liballegro5-dev +liballegro-ttf5-dev +libevent-dev +libpuzzle1 +libssh2-1 +liblinear-dev +libsvm3 +libxml2-dev +libc6 +libblas3 +liblapack3 +freetds-dev +ocl-icd-opencl-dev +libode-dev +libopenal-dev +libalut-dev +libfann-dev +libglpk-dev +libgsl0-dev +libpango1.0-dev +libplplot-dev +libpng-dev +libportaudio2 +libproj-dev +pslib1 +librabbitmq-dev +r-mathlib +libreadline-dev +librrd-dev +librsvg2-2 +zlib1g-dev +libsane-dev +libsdl2-image-2.0-0 +libsdl2-ttf-2.0-0 +libsoil-dev +libsdl1.2-dev +libtesseract-dev +libtidy-dev +libtokyocabinet-dev +libwayland-dev +libsoup2.4-1 +libwebkit2gtk-4.0-dev +libxkbcommon-dev +libassimp-dev +libsdl-mixer1.2 +default-libmysqlclient-dev +unixodbc-dev +libpq-dev +libsqlite0-dev +libncursesw5 +libxrandr-dev +libbluetooth-dev +libsdl2-gfx-1.0-0 +libev4 +libleveldb-dev +libnet1-dev +libsdl-ttf2.0-0 +liblmdb-dev +libmagic-dev +libflac8 +libasound2 +libmpg123-0 +libvorbisfile3 +libpcap0.8-dev +triplea +gramps +r-base-core +libsmokeqt4-dev +libqt4-dev +libsmokeqtgui4-3 +libssl-dev +libsnappy-dev diff --git a/debian-setup/package.sh b/debian-setup/package.sh new file mode 100644 index 0000000..eb27148 --- /dev/null +++ b/debian-setup/package.sh @@ -0,0 +1,10 @@ +#!/bin/bash + + +mkdir -p done +for f in $(cat debian-9-packages.txt);do + if ! [ -f done/$f ];then + sudo apt-get -y install $f + touch done/$f + fi +done diff --git a/depcheck.lisp b/depcheck.lisp index 74c29f4..17cd23a 100644 --- a/depcheck.lisp +++ b/depcheck.lisp @@ -5,6 +5,10 @@ (in-package #:depcheck) +(eval-when (:compile-toplevel :load-toplevel :execute) + (when (find-symbol "SBCL-HOMEDIR-PATHNAME" "SB-SYS") + (pushnew :sbcl-homedir-pathname *features*))) + (defvar *direct-dependencies* nil) (defun load-asdf-system-table (file) @@ -115,6 +119,38 @@ ;;(check-attribute 'asdf:system-license :license) (check-attribute 'asdf:system-author :author)))) +(defun child-system-p (maybe-parent-name maybe-child-name) + (and (not (equal maybe-parent-name maybe-child-name)) + (equal maybe-parent-name (asdf:primary-system-name maybe-child-name)))) + +(defun system-direct-dependencies (system-designator) + (mapcar 'normalize-dependency + (asdf:system-depends-on (asdf:find-system system-designator)))) + +(defun replace-inferred-system-deps-1 (system-name dependencies) + (let* ((inferred-children-deps + (remove-if-not (lambda (name) (child-system-p system-name name)) + dependencies)) + (deps-of-children + (reduce 'append + (mapcar 'system-direct-dependencies inferred-children-deps) + :initial-value nil))) + (values + (append (set-difference dependencies inferred-children-deps) + deps-of-children) + (null inferred-children-deps)))) + +(defun replace-inferred-system-deps (system-name dependencies) + (let ((external-deps dependencies)) + (loop + (multiple-value-bind (new-dependencies donep) + (replace-inferred-system-deps-1 system-name external-deps) + (setf external-deps (remove-duplicates new-dependencies + :test #'equal)) + (when donep + (return)))) + external-deps)) + (defun compute-dependencies (system-file system-name) (let* ((asdf:*system-definition-search-functions* (list #-asdf3 'asdf::sysdef-find-asdf @@ -131,6 +167,10 @@ (when (equalp system-file system-name) (setf dependencies *implied-dependencies*))) (asdf:oos 'asdf:load-op system-name) + (when (and (typep (asdf:find-system system-name) 'asdf:package-inferred-system) + (equalp system-name system-file)) + (setf *direct-dependencies* + (replace-inferred-system-deps system-name *direct-dependencies*))) (setf dependencies (remove-duplicates (append *direct-dependencies* dependencies) :test #'equalp)) @@ -196,11 +236,15 @@ (when (equalp (second argv) "--sbcl-version") (format t "~A~%" (lisp-implementation-version)) (sb-ext:exit :code 0)) + (when (getenv "DEPCHECK_HIDEBUG") + (sb-ext:restrict-compiler-policy 'debug 3) + (sb-ext:restrict-compiler-policy 'safety 3)) (unless (getenv "DEPCHECK_DEBUG") (sb-ext:disable-debugger)) (setenv "SBCL_HOME" (load-time-value (directory-namestring sb-int::*core-string*))) + (setf sb-sys::*sbcl-homedir-pathname* (sb-impl::%sbcl-homedir-pathname)) #+nil (setenv "CC" "gcc") (eval *load-op-wrapper*) diff --git a/dist-cache.lisp b/dist-cache.lisp index 441904f..bc10580 100644 --- a/dist-cache.lisp +++ b/dist-cache.lisp @@ -2,6 +2,24 @@ (in-package #:quicklisp-controller) +(defvar *system-file-index-file* + #p"quicklisp-controller:dist;system-file-index") + +(defun clear-tar-cache () + (rm-rf "quicklisp-controller:dist;tar-cache;")) + +(defun clear-build-cache () + (rm-rf "quicklisp-controller:dist;build-cache;") + (rm-rf *system-file-index-file*)) + +(defun clear-build-artifacts () + (rm-rf "quicklisp-controller:dist;build-artifacts;")) + +(defun clear-dist-caches () + (clear-tar-cache) + (clear-build-cache) + (clear-build-artifacts)) + (defgeneric find-cached-release-tarball (source) (:method (source) (let* ((wild @@ -57,11 +75,14 @@ (defun build-relative (pathname source) - (merge-pathnames pathname (ensure-cached-build-directory source))) + (merge-pathnames pathname + (merge-pathnames + (make-pathname :directory (list :relative (name source))) + (translate-logical-pathname "quicklisp-controller:dist;build-artifacts;")))) ;;; System files -(defun blacklist-table (name) +(defun excluded-systems-table (name) (let* ((pathname (merge-logical (make-pathname :name name) #p"quicklisp-controller:projects;qlc-meta;template.txt")) (lines (and (probe-file pathname) (config-file-lines pathname))) @@ -69,16 +90,16 @@ (dolist (line lines table) (setf (gethash line table) t)))) -(defun blacklist-list (name) +(defun excluded-systems-list (name) (let ((pathname (merge-logical (make-pathname :name name) #p"quicklisp-controller:projects;qlc-meta;template.txt"))) (when (probe-file pathname) (config-file-lines pathname)))) -(defun make-blacklister (source) +(defun make-system-excluder (source) (lambda (system-file) - (let ((bad-patterns (blacklist-list "system-pathname-blacklist")) - (bad-combos (blacklist-table "blacklist")) + (let ((bad-patterns (excluded-systems-list "excluded-system-pathnames")) + (bad-combos (excluded-systems-table "excluded-systems")) (combo-key (format nil "~A ~A" (project-name source) (pathname-name system-file)))) @@ -87,10 +108,11 @@ (search string (namestring system-file))) bad-patterns))))) -(defun blacklistedp (source system-file) - "Is SYSTEM-FILE for SOURCE somehow forbidden, e.g. " - (let ((bad-patterns (blacklist-list "system-pathname-blacklist")) - (bad-combos (blacklist-table "blacklist")) +(defun excluded-system-p (source system-file) + "Is SYSTEM-FILE for SOURCE excluded through a pathname exclusion +list or a system name exclusion list?" + (let ((bad-patterns (excluded-systems-list "excluded-system-pathnames")) + (bad-combos (excluded-systems-table "excluded-systems")) (combo-key (format nil "~A ~A" (project-name source) (pathname-name system-file))) @@ -111,12 +133,12 @@ (defun build-system-files (source) "Return a list of system files in the build directory of SOURCE." (setf source (source-designator source)) - (let* ((blacklist-fun (make-blacklister source)) + (let* ((excluded-system-fun (make-system-excluder source)) (base (ensure-cached-build-directory source)) (wild (merge-pathnames "**/*.asd" base)) (files (directory wild))) (mapcan (lambda (file) - (unless (funcall blacklist-fun file) + (unless (funcall excluded-system-fun file) (when (find-if #'upper-case-p (file-namestring file)) (error "Mixed-case system file ~A cannot be used" file)) @@ -150,20 +172,20 @@ if needed." (setf source (source-designator source)) (let ((files (ensure-build-system-files source))) (remove-if (lambda (file) - (blacklistedp source file)) + (excluded-system-p source file)) files))) (defun system-names (source) (setf source (source-designator source)) (mapcar 'pathname-name (system-files source))) - (defun asdf-systems-table () "Return a hash table that maps system names to system files." (let ((table (make-string-table))) ;; Add SBCL contribs first - (let ((contrib-system-files - (directory "/usr/local/lib/sbcl/contrib/*.asd"))) + (let* ((base (sb-int:sbcl-homedir-pathname)) + (contrib-system-files + (directory (merge-pathnames "**/*.asd" base)))) (dolist (file contrib-system-files) (setf (gethash (pathname-name file) table) file))) (map-sources @@ -184,6 +206,7 @@ if needed." :direction :output :if-exists :supersede) (maphash (lambda (system-name system-file) + (declare (ignore system-name)) (format stream "~A~%" (enough-namestring system-file (translate-logical-pathname file)))) @@ -204,9 +227,6 @@ if needed." table)) -(defvar *system-file-index-file* - #p"quicklisp-controller:dist;system-file-index") - (defun update-system-file-index () (let ((table (asdf-systems-table))) (save-asdf-system-table table *system-file-index-file*) @@ -366,7 +386,7 @@ their name does not match the system file name." (with-system-index (dolist (system-file-name (system-names source)) (dolist (system (ignore-errors (system-defined-systems system-file-name))) - (unless (blacklistedp source system) + (unless (excluded-system-p source system) (funcall fun system-file-name system)))))) (defun acceptable-system-name (name) @@ -394,6 +414,7 @@ structure \(SYSTEM-FILE-NAME SYSTEM-NAME &REST DEPENDENCIES). " (let ((winners '()) (timing-file (timing-file source)) (start-time (get-universal-time))) + (ensure-directories-exist timing-file) (map-source-systems source (lambda (system-name system) @@ -434,7 +455,7 @@ structure \(SYSTEM-FILE-NAME SYSTEM-NAME &REST DEPENDENCIES). " (defun build-duration (source) (destructuring-bind (&key start-time end-time) (timing-data source) - (if start-time + (if (and start-time end-time) (- end-time start-time) -1))) diff --git a/failure-report.css b/failure-report.css index 2321b11..0fe6532 100644 --- a/failure-report.css +++ b/failure-report.css @@ -1,3 +1,11 @@ +pre.snippet { + overflow-wrap: break-word; + color: black; + font-weight: normal; + white-space: pre-wrap; + width: 75em; +} + .failing-system pre { color: #111; margin-left: 2em; diff --git a/git.lisp b/git.lisp index 8b73543..469677d 100644 --- a/git.lisp +++ b/git.lisp @@ -33,12 +33,20 @@ (message (or commit-message (commit-message source)))) (unless (or unclean (clean-stage-p)) - (error "Stage isn't clean")) + (cerror "Continue anyway" "Stage isn't clean")) (in-projects-directory (run "git" "add" (pathname file)) (run "git" "commit" "-m" message)) (list :committed (name source) :with message))) +(defmacro with-issue-number ((var source) &body body) + (let ((source-var (copy-symbol 'source))) + `(let* ((,source-var (source-designator ,source)) + (,var (github-issue-number ,source-var))) + (unless ,var + (error "Can't find issue number for ~A" ,source-var)) + ,@body))) + (defun set-issue-label (source label) (setf source (source-designator source)) (let ((issue-number (github-issue-number source))) @@ -49,6 +57,13 @@ :number issue-number :body (githappy:js "labels" (list label))))) +(defun comment-on-issue (source comment) + (with-issue-number (number source) + (githappy:create-repo-issue-comment :body (githappy:js "body" comment) + :issue-number number + :repo "quicklisp-projects" + :owner "quicklisp"))) + (defun mark-canbuild (source) (set-issue-label source "canbuild")) diff --git a/html-failure-report.lisp b/html-failure-report.lisp index 89fc002..ab2744a 100644 --- a/html-failure-report.lisp +++ b/html-failure-report.lisp @@ -82,6 +82,160 @@ the string is returned unchanged." (when substituted (return substituted))))) +(defun reconstitute (string patterns) + (let ((result (maybe-reconstitute string patterns))) + (and (not (eq result string)) + result))) + +;;; Linking to VCS sources from log lines + +(defun parse-vcs-source-log-line (log-line) + "Return a plist of info about log-line." + (when (and (search "dist/build-cache/" log-line) + (not (search ".cache/common-lisp" log-line))) + (ppcre:register-groups-bind (project-name full-path) + ("dist/build-cache/(.*?)/(.*$)" log-line) + (let* ((pos 0) + (second-slash + (dotimes (i 2 pos) + (setf pos (position #\/ full-path :start (1+ pos))))) + (end (and second-slash (position #\& full-path :start second-slash)))) + (let ((source (find-source project-name))) + (when (and source second-slash) + (let* ((path (subseq full-path (1+ second-slash) end)) + (start (search path log-line)) + (line-number nil)) + (ppcre:register-groups-bind ((#'parse-integer log-line-number)) ("Line: (\\d+)" log-line) + (setf line-number log-line-number)) + (list :source source + :path path + :line-number line-number + :path-bounds (cons start (+ start (length path))))))))))) + +(defparameter *location-base-substitutions* + '(("(https://github.com/.*?/.*?)\\.git" 0 "/blob/") + ("(https://.*gitlab.*)\\.git$" 0 "/blob/") + ("(https://bitbucket.org/.*?)\\.git$" 0 "/src/") + ("(http://dwim.hu/live/.*$)" 0) + ("^(https:.*notabug.*)\.git$" 0 "/src/"))) + +(defun location-base (location) + (reconstitute location *location-base-substitutions*)) + +(defun source-branch (source) + (typecase source + (tagged-mixin + (tag-data source)) + (git-source + "master") + (t + nil))) + +(defun source-file-link-base (source) + (let ((base (location-base (location source)))) + (when base + (format nil "~A~@[~A/~]" base (source-branch source))))) + +(defun source-file-link (source path line-number) + (let ((base (source-file-link-base source))) + (when base + (format nil "~A~A~@[#L~A~]" base path line-number)))) + +(defun link-subseq (line link bounds) + (destructuring-bind (start . end) + bounds + (concatenate 'string + (subseq line 0 start) + "" + (subseq line start end) + "" + (subseq line end)))) + + +;;; Scraping error logs for the real problem + +(defstruct peekstream + stream + buffered-line) + +(defun next-line (peekstream) + (if (peekstream-buffered-line peekstream) + (shiftf (peekstream-buffered-line peekstream) nil) + (read-line (peekstream-stream peekstream) nil))) + +(defun peek-line (peekstream) + (or (peekstream-buffered-line peekstream) + (setf (peekstream-buffered-line peekstream) + (read-line (peekstream-stream peekstream) nil)))) + +(defun unread-line (line peekstream) + (setf (peekstream-buffered-line peekstream) line)) + +(defun =line-matches (pattern) + (let ((scanner (ppcre:create-scanner pattern))) + (lambda (line) + (ppcre:scan scanner line)))) + +(defun callfun (object) + (lambda (fun) + (funcall fun object))) + +(defun =or (&rest funs) + (lambda (object) + (some (callfun object) funs))) + +(defun =not (fun) + (complement fun)) + +(defun skip-text-until (fun stream) + (loop for line = (peek-line stream) + while line + do + (cond ((funcall fun line) + (return line)) + (t + (next-line stream))))) + +(defun collect-text-while (fun stream) + (with-output-to-string (s) + (loop for line = (peek-line stream) + while (and line (funcall fun line)) + do (write-line (next-line stream) s)))) + +(defun collect-text-between (start-fun end-fun stream) + (let* ((first-line (skip-text-until start-fun stream)) + (rest (and first-line (next-line stream) (collect-text-while (=not end-fun) stream)))) + (when first-line + (concatenate 'string + first-line #(#\newline) + rest)))) + +(defun extract-warnings-and-errors (file) + "Scrape warnings and errors out of the logfile FILE. Repeated +consecutive strings are coalesced into a single string to avoid +redundant info." + (flet ((coalesce-consecutive-strings (strings) + (let ((out '())) + (dolist (string strings (nreverse out)) + (unless (equal (first out) string) + (push string out)))))) + (coalesce-consecutive-strings + (with-open-file (stream file) + (let ((pstream (make-peekstream :stream stream))) + (let (result) + (loop + (let ((match (collect-text-between (=or (=line-matches "^; caught (COMMON-LISP:)?(WARNING|ERROR):") + (=line-matches "^Unhandled")) + (=not (=line-matches "^;")) + pstream))) + (if match + (push match result) + (return (nreverse result))))))))))) + + + ;;; Posting to S3 (defun report-publishing-enabled-p () @@ -90,8 +244,8 @@ the string is returned unchanged." (defun content-type (file) (cond ((equalp (pathname-type file) "css") "text/css") - ((equalp (pathname-type file) "rss") - "application/rss+xml") + ((equalp (pathname-type file) "rss") + "application/rss+xml") (t "text/html"))) @@ -124,9 +278,9 @@ the string is returned unchanged." (defgeneric full-failure-report-url (object) (:method (object) (format nil "http://~A/~A~A" - *failtail-bucket* - (report-prefix) - (failure-report-url object)))) + *failtail-bucket* + (report-prefix) + (failure-report-url object)))) (defgeneric failure-report-html-file (base object)) (defgeneric stylesheet-path (object)) @@ -136,6 +290,8 @@ the string is returned unchanged." (defgeneric write-html-failure-report-content (object stream)) (defgeneric write-html-failure-report-footer (object stream)) +(defgeneric failure-impact (object)) + (defmethod failure-report-html-file (base object) (relative-to base (failure-report-url object))) @@ -153,34 +309,52 @@ the string is returned unchanged." :reader source) (failure-log-file :initarg :failure-log-file - :reader failure-log-file))) + :reader failure-log-file) + (warnings-and-errors + :reader warnings-and-errors) + (breaks + :accessor breaks + :initform nil) + (broken-by + :accessor broken-by + :initform nil))) (defmethod print-object ((object failing-system) stream) (print-unreadable-object (object stream :type t) (write-string (system-name object) stream))) +(defmethod failure-impact ((object failing-system)) + (length (breaks object))) + +(defmethod slot-unbound ((class t) (instance failing-system) (slot (eql 'warnings-and-errors))) + (setf (slot-value instance 'warnings-and-errors) + (extract-warnings-and-errors (failure-log-file instance)))) + +(defun broken-by-name (failing-system) + (let* ((unhandled-log-line + (find "Unhandled" (warnings-and-errors failing-system) + :test #'search)) + (responsible-system-name + (ppcre:register-groups-bind (system-name) + ("^Unhandled.*while compiling.*SOURCE-FILE \"(.*?)\"" + unhandled-log-line) + system-name))) + (or responsible-system-name + (system-name failing-system)))) + (defmethod new-failure-p ((object failing-system)) (let* ((dist (ql-dist:find-dist "quicklisp")) - (system (ql-dist:find-system-in-dist (system-name object) dist))) - (not (not system)))) + (existing-system + (ql-dist:find-system-in-dist (system-name object) dist))) + (or (not (not existing-system)) + (< (days-old (source object)) 30)))) (defmethod failure-data ((source upstream-source)) - (let ((result '())) - (map-source-systems - source - (lambda (system-file-name system-name) - (write-char #\. *trace-output*) - (force-output *trace-output*) - (let ((file (winfail-file "fail" source system-file-name system-name))) - (when (probe-file file) - (push (make-instance 'failing-system - :system-name system-name - :system-file-name - system-file-name - :source source - :failure-log-file file) - result))))) - result)) + (remove (name source) + (failing-systems) + :test-not 'string= + :key (lambda (system) + (name (source system))))) (defmethod name ((object failing-system)) (name (source object))) @@ -208,6 +382,9 @@ the string is returned unchanged." (name (source object)) (length (failure-data object))))) +(defmethod failure-impact ((object failing-source)) + (reduce #'+ (mapcar #'failure-impact (failure-data object)))) + (defmethod source-link ((source failing-source)) (source-link (source source))) @@ -260,61 +437,81 @@ source is found that matches the filename, return nil." ;; reason), so don't try to make a failing-source in that case. (let ((source (find-source source-name))) (when source - (make-instance 'failing-system - :source (find-source source-name) - :failure-log-file failure-file - :system-file-name system-file-name - :system-name (decode-string-from-filesystem - failing-system)))))) + (make-instance 'failing-system + :source (find-source source-name) + :failure-log-file failure-file + :system-file-name system-file-name + :system-name (decode-string-from-filesystem + failing-system)))))) (defun failing-source-log-files () - (let* ((base (translate-logical-pathname "quicklisp-controller:dist;build-cache;")) - (fail-wild (merge-pathnames "**/fail_*_*_*.txt" base))) + (let* ((base (translate-logical-pathname "quicklisp-controller:dist;build-artifacts;")) + (fail-wild (merge-pathnames "**/fail_*_*_*.txt" base))) (directory fail-wild))) (defun failing-systems () - (remove nil - (mapcar #'parse-failure-file-name - (failing-source-log-files)))) + ;; This is the best way to get failure info, because it populates + ;; useful failure cross-reference data. + (let* ((systems (remove nil + (mapcar #'parse-failure-file-name + (failing-source-log-files)))) + (table (make-hash-table :test 'equal))) + (dolist (system systems) + (setf (gethash (system-name system) table) system)) + (dolist (system systems) + (let ((broken-by (or (gethash (broken-by-name system) table) + system))) + (unless (eq broken-by system) + (setf (broken-by system) + broken-by) + (push system (breaks broken-by))))) + systems)) + +(defun who-is-broken-by (name) + (remove name (failing-systems) + :test-not #'string= + :key (lambda (failing-system) + (and (broken-by failing-system) + (system-name (broken-by failing-system)))))) (defun failure-log-failure-report () "Scan the failure log files of all projects to produce a failure report." (let ((systems (make-hash-table :test 'equal))) (flet ((fsource (source) - (or (gethash (name source) systems) - (setf (gethash (name source) systems) - (make-instance 'failing-source - :failure-data nil - :source source))))) + (or (gethash (name source) systems) + (setf (gethash (name source) systems) + (make-instance 'failing-source + :failure-data nil + :source source))))) (let ((table (make-hash-table :test 'eq)) - (systems (failing-systems)) - (report (make-instance 'failure-report - :failure-data '()))) - (dolist (system systems) - (let ((key (fsource (source system)))) - (push system (gethash key table)))) - (maphash (lambda (failing-source failing-systems) - (setf (failure-data failing-source) failing-systems) - (push failing-source (failure-data report))) - table) - report)))) + (systems (failing-systems)) + (report (make-instance 'failure-report + :failure-data '()))) + (dolist (system systems) + (let ((key (fsource (source system)))) + (push system (gethash key table)))) + (maphash (lambda (failing-source failing-systems) + (setf (failure-data failing-source) failing-systems) + (push failing-source (failure-data report))) + table) + report)))) (defmethod failure-data ((object (eql t))) (failure-log-failure-report)) (defparameter *log-lines-that-are-boring* (mapcar 'ppcre:create-scanner - '("^WARNING:"))) + '("^WARNING:"))) (defparameter *log-lines-to-highlight* (mapcar 'ppcre:create-scanner '("^; caught (WARNING|ERROR):" " READ error during" - "^Backtrace for" + "^Backtrace for" "^Unhandled"))) (defparameter *failure-log-reconstitution-patterns* - '(("(The ANSI Standard, Section )([0-9.]*)" + '(("(^.*The ANSI Standard, Section )([0-9.]*)" 0 "" 1 ""))) (defun failure-log-reconstitute-line (line) @@ -328,6 +525,13 @@ source is found that matches the filename, return nil." (loop for scanner in *log-lines-that-are-boring* thereis (ppcre:scan scanner line))) +(defun failure-snippet (object) + (etypecase object + (failing-system + (format nil "~{~A~^...~%~}~%" + (extract-warnings-and-errors (failure-log-file object)))) + (failing-source + (format nil "~{~A~}" (mapcar #'failure-snippet (failure-data object)))))) (defmethod write-html-failure-report-header (object stream) (format stream "~A~ @@ -363,18 +567,25 @@ source is found that matches the filename, return nil." (loop for line = (read-line log-stream nil) while line do - (setf line (cl-who:escape-string line)) - (cond ((highlighted-log-line-p line) - (write-string "" stream) - (write-string line stream) - (write-string "" stream) - (terpri stream)) - ((boring-log-line-p line) - (format stream "~A~%" line)) - (t - (write-line line stream))))) + (setf line (failure-log-reconstitute-line (cl-who:escape-string line))) + (let ((upstream-info (parse-vcs-source-log-line line))) + (when upstream-info + (destructuring-bind (&key source path path-bounds line-number &allow-other-keys) + upstream-info + (let ((link (source-file-link source path line-number))) + (when link + (setf line (link-subseq line link path-bounds))))))) + (cond ((highlighted-log-line-p line) + (write-string "" stream) + (write-string line stream) + (write-string "" stream) + (terpri stream)) + ((boring-log-line-p line) + (format stream "~A~%" line)) + (t + (write-line line stream)))) (format stream "") - (format stream "~%")) + (format stream "~%"))) (defmethod write-html-failure-report-content ((source failing-source) stream) (dolist (system (failure-data source)) @@ -395,22 +606,45 @@ source is found that matches the filename, return nil." (new (remove-if-not #'new-failure-p sources)) (old (remove-if #'new-failure-p sources))) (flet ((show (sources) - (dolist (source sources) + (dolist (source (sort (copy-seq sources) #'string< :key #'name)) (let ((link (source-link source))) (format stream " ~A:
" (new-failure-p source) (name source)) + (let ((age (source-cache-age-or-nil (source source)))) + (when age + (format stream "last modified ~A ago
~%" (how-long-ago age)))) (if link (format stream "~A" link link) (format stream "~A" (location (source source)))) (format stream "~%") (format stream "~%")))) + (dolist (system (sort (copy-seq (failure-data source)) #'string< :key #'system-name)) + (let ((responsible (broken-by system)) + (system-name (system-name system))) + (format stream " ~A" + (new-failure-p system) + system-name + (failure-report-url system) + system-name) + (when responsible + (format stream " caused by ~A~%" + (system-name responsible) + (system-name responsible))) + (when (breaks system) + (format stream "
Breaks: ") + (dolist (broken (breaks system)) + (format stream "~A " + (system-name broken) + (system-name broken))) + (format stream "
")) + (unless responsible + (format stream "
~A
" + (cl-who:escape-string (failure-snippet system)))) + (when responsible + (format stream "
")) + (format stream "~%")))) + (format stream "")))) (show new) (format stream "

") (show old))) @@ -422,6 +656,9 @@ source is found that matches the filename, return nil." (let ((link (source-link (source object)))) (when link (format stream "
  • site: ~A~%" link link))) + (let ((age (source-cache-age-or-nil (source object)))) + (when age + (format stream "
  • last updated: ~A ago" (how-long-ago age)))) (format stream "~%") (format stream "

    ~A~%" (versions-and-such))) diff --git a/irepl.lisp b/irepl.lisp index c66434b..49bc18a 100644 --- a/irepl.lisp +++ b/irepl.lisp @@ -126,12 +126,15 @@ (define-irepl-command skip (let ((index (position-if-not (=jref '("labels" :* "name")) - (all-issues *irepl-state*)))) + (all-issues *irepl-state*)))) (if index - (progn - (setf (issue-index *irepl-state*) index) - (invoke-irepl-command 'show)) - (format t "; No issue without labels~%")))) + (progn + (setf (issue-index *irepl-state*) index) + (invoke-irepl-command 'show)) + (format t "; No issue without labels~%")))) + +(define-irepl-command failtail + (failtail)) (defvar *irepl-guess-patterns* "https://github.com/") @@ -142,9 +145,15 @@ :canbuild)) (define-irepl-command cantbuild - (when *last-source* - (mark-cantbuild *last-source*) - :cantbuild)) + (cond (*last-source* + (mark-cantbuild *last-source*) + (let ((log-url (publish-source-failure *last-source*))) + (comment-on-issue *last-source* + (format nil "Failure log here: ~A" + log-url))) + :cantbuild) + (t + (warn "No last-source defined")))) (define-irepl-command commit (when *last-source* diff --git a/misc.lisp b/misc.lisp index fd0cb77..52c4f7f 100644 --- a/misc.lisp +++ b/misc.lisp @@ -6,6 +6,11 @@ (run "rm" "-rf" (merge-pathnames ".cache/common-lisp/" (user-homedir-pathname)))) +(defun clear-all-caches () + (clear-fasl-cache) + (clear-dist-caches) + (clear-system-file-magic-cache)) + (defun system-from-release (system-name dist) (let* ((dist (ql-dist:dist dist)) (system (ql-dist:find-system-in-dist system-name dist)) @@ -162,15 +167,27 @@ (check-for-program program))) (defun crank (&optional (source *last-source*)) + (setf source (source-designator source)) (check-critical-programs) (unless (source-designator source) (warn "Not a known source -- ~S" source) (return-from crank nil)) (setf *last-source* source) (update-system-file-index) - (let ((wins (find-more-winning-systems source))) - (list :fails (missing-components source) - :wins wins))) + (find-more-winning-systems source) + (let ((fails (failure-data source))) + (when fails + (format t "FAILURES:~%") + (dolist (fail fails) + (cond ((broken-by fail) + (let ((responsible-system (broken-by fail))) + (format t "System ~A broken by ~A~%" + (system-name fail) + (system-name responsible-system)))) + (t + (format t "~S:~%~A" + (system-name fail) + (failure-snippet fail)))))))) (defun source-pathname (project-name) (let ((directory `(:relative "quicklisp-controller" @@ -281,7 +298,7 @@ (tail-file file n))) (defvar *failtail-credentials-file* - #p"quicklisp-controller:failtail-credentials.txt") + #p"quicklisp-controller:dist;failtail-credentials.txt") (defvar *failtail-credentials* (zs3:file-credentials *failtail-credentials-file*)) @@ -393,7 +410,7 @@ (error "Can't guess project name")) (when (equal name "") (error "Name can't be empty")) - (let ((file (project-source-filename name))) + (let ((file (project-name-source-file name))) (restart-case (when (probe-file file) (error "Project already has a file")) @@ -530,13 +547,16 @@ (defun =location~ (substring) (lambda (source) - (search substring (location source)))) + (ppcre:scan substring (location source)))) (defun http-to-https (source) (let ((location (ppcre:regex-replace "https://" (location source) "https://"))) (format nil "https ~A" location))) (defun rewrite-sources (match-fun new-source-fun) + "Rewrite any source that returns TRUE from MATCH-FUN with +NEW-SOURCE-FUN, which should return a single line for use in the +source's source.txt file. Useful for bulk-updating sources." (map-sources (lambda (source) (when (funcall match-fun source) @@ -544,3 +564,34 @@ (when new-source (setf (first-line (source-file source)) new-source))))))) + + +;;; Showing update failures from a recrank log + +(defun starts-with (subseq seq) + (and (<= (length subseq) (length seq)) + (every #'equal subseq seq))) + +(defun crank-projects (crank-logfile) + (with-open-file (stream crank-logfile) + (loop for line = (read-line stream nil) + while line + when (and line (starts-with "* " line)) + collect (subseq line 2)))) + +(defun crank-failures (crank-logfile) + (dolist (project (crank-projects crank-logfile)) + (format t ";;; ~A~%" project) + (with-simple-restart (skip "Skip ~A" project) + (update-source-cache (find-source project))))) + +;;; Recrank only failing systems + +(defun recrank-failing-systems () + (map nil + (lambda (source) + (format *trace-output* "~&; XXX updating and cranking ~A~%" source) + (update-and-crank source)) + (remove-duplicates (mapcar 'source (failing-systems)) + :key 'name + :test 'equal))) diff --git a/quicklisp-controller.asd b/quicklisp-controller.asd index 5bfde62..5bba6db 100644 --- a/quicklisp-controller.asd +++ b/quicklisp-controller.asd @@ -12,17 +12,16 @@ #:drakma #:yason #:function-cache - #:trivial-utf-8 + #:trivial-utf-8 #:ironclad #:lparallel - #:cl-who - #:ubiquitous + #:cl-who #:githappy #:project-info - #:westbrook) + #:westbrook) :serial t :components ((:file "tarhash") - (:file "github-issues") + (:file "github-issues") (:file "package") (:file "config") (:file "logical-host") @@ -30,7 +29,7 @@ (:file "utils") (:file "setup") (:file "upstream") - (:file "system-file-magic-cache") + (:file "system-file-magic-cache") (:file "dist-cache") (:file "upstream-vcs") (:file "upstream-http") @@ -39,6 +38,7 @@ (:file "upstream-darcs") (:file "upstream-git") (:file "upstream-github") + (:file "upstream-gitlab") (:file "upstream-mercurial") (:file "upstream-svn") (:file "upstream-bzr") @@ -52,11 +52,12 @@ (:file "update-client-version") (:file "descriptions") (:file "ng-indexes") - (:file "git") - (:file "html-failure-report") - (:file "rss-failure-feeds") + (:file "git") + (:file "html-failure-report") + (:file "rss-failure-feeds") (:file "recrank") - (:file "irepl"))) + (:file "irepl") + (:file "submit-bug-report"))) (defpackage #:quicklisp-controller-config (:use) diff --git a/recrank.lisp b/recrank.lisp index 793417f..c37b064 100644 --- a/recrank.lisp +++ b/recrank.lisp @@ -8,6 +8,7 @@ (file #p"quicklisp:tmp;update-failures.txt")) (clear-fasl-cache) (preflight) + (clear-dist-caches) (when update (update-what-you-can :file file :parallel parallel) (when (and file *report-to-email*) @@ -16,9 +17,6 @@ :subject "Quicklisp update failures" :from *report-to-email* :to *report-to-email*)))) - (run "rm" "-rf" - (native-namestring - (translate-logical-pathname #p"quicklisp-controller:dist;"))) (ensure-what-wins-you-can) (when report (with-skipping diff --git a/submit-bug-report.lisp b/submit-bug-report.lisp new file mode 100644 index 0000000..204cb53 --- /dev/null +++ b/submit-bug-report.lisp @@ -0,0 +1,87 @@ +;;;; submit-bug-report.lisp +;;;; +;;;; Semiautomatically create bug reports to send to projects for +;;;; systems that fail. +;;;; +;;;; This is based around the data in a failing-system object, defined +;;;; in html-failure-report.lisp +;;;; + +(in-package #:quicklisp-controller) + +(defun github-source-p (source) + (search "/github.com/" (location source))) + +(deftype github-source () + `(satisfies github-source-p)) + +(defun github-owner (failing-system) + (nth-value 0 (github-owner-and-repo (location (source failing-system))))) + +(defun github-repo (failing-system) + (nth-value 1 (github-owner-and-repo (location (source failing-system))))) + +(defun submit-github-issue (owner repo title body) + (githappy::create-repo-issue + :owner owner + :repo repo + :body (githappy:js "title" title "body" body))) + + +(defun existing-bug-reports (source) + (multiple-value-bind (owner repo) + (github-owner-and-repo (location source)) + (unless owner + (error "Not a github repo")) + (let* ((response (githappy:repo-issues :owner owner :repo repo :per-page 100)) + (json (githappy:json response))) + (mapcan + (lambda (issue) + (when (equal "quicklisp" (githappy:jref issue '("user" "login"))) + (list (list :number (githappy:jref issue "number") + :title (githappy:jref issue "title"))))) + json)))) + +(defun blameless-for-failure-p (failing-source) + (null (remove-if #'broken-by (failure-data failing-source)))) + +(defun bug-report-body (failing-source &key log-link) + (with-output-to-string (s) + (format s "Building with ~A for quicklisp dist creation.~%~%" + (versions-and-such)) + (format s "Trying to build commit id ~A~%~%" (commit-id (source failing-source))) + (dolist (system (failure-data failing-source)) + (format s "*~A* fails to build" (system-name system)) + (if (broken-by system) + (format s " because of a failure in _~A_.~%~%" + (system-name (broken-by system))) + (format s " with the following error:~%~%```~%~A~&```~%~%" + (failure-snippet system)))) + (when log-link + (format s "[Full log here](~A)~%~%" log-link)))) + +(defun report-bug-stuff (source) + (setf source (source-designator source)) + (let* ((failing-source (find-failing-source source)) + (log-link (publish-source-failure source)) + (body (bug-report-body failing-source :log-link log-link)) + (title "Some systems failed to build for Quicklisp dist")) + (list :title title + :body body))) + +(defun report-bug-in (source) + (setf source (source-designator source)) + (let* ((failing-source (find-failing-source source)) + (log-link (publish-source-failure source)) + (body (bug-report-body failing-source :log-link log-link))) + (multiple-value-bind (owner repo) + (github-owner-and-repo (location source)) + (format t "Posting bug report for ~A~%~%" source) + (format t "~A" body) + (let ((existing (existing-bug-reports (source failing-source)))) + (when existing + (format t "WARNING: BUGS ALREADY SUBMITTED BY quicklisp:~%~{ ~A~%~}~%" + existing))) + (when (ql-util:press-enter-to-continue) + (submit-github-issue owner repo "Some systems failed to build for Quicklisp dist" body))))) + diff --git a/system-file-magic.lisp b/system-file-magic.lisp index a3f283b..f40407a 100644 --- a/system-file-magic.lisp +++ b/system-file-magic.lisp @@ -64,6 +64,18 @@ (format stream "~S~%~%" sexp))))) +(defun setenv (name value) + (let ((r + (sb-alien:alien-funcall + (sb-alien:extern-alien "setenv" + (sb-alien:function + sb-alien:int (sb-alien:c-string :not-null t) + (sb-alien:c-string :not-null t) sb-alien:int)) + name value 1))) + (if (minusp r) + (error "setenv") + r))) + (defun main (argv) (setf *package* (find-package :keyword)) (sb-ext:disable-debugger) @@ -71,6 +83,11 @@ ;; (load-time-value ;; (directory-namestring sb-int::*core-string*)) ;; 1) + (setenv "SBCL_HOME" + (load-time-value + (directory-namestring sb-int::*core-string*))) + (setf sb-sys::*sbcl-homedir-pathname* (sb-impl::%sbcl-homedir-pathname)) + (destructuring-bind (index-file system-name output-file &optional project-name description-file) (rest argv) diff --git a/update-failure-report.lisp b/update-failure-report.lisp new file mode 100644 index 0000000..0c0a873 --- /dev/null +++ b/update-failure-report.lisp @@ -0,0 +1,21 @@ +(in-package #:quicklisp-controller) + +(defun latest-build-log-file () + (let ((logs (directory (asdf:system-relative-pathname "quicklisp-controller" + #p"logs/**/*.txt")))) + (first (last (sort logs #'string< :key #'namestring ))))) + +(defun collect-matching (fun stream) + (loop for line = (next-line stream) + while line + when (funcall fun line) + collect line)) + +(defun extract-failed-updates (file) + (with-open-file (stream file) + (mapcar (lambda (line) + (find-source (second (split-spaces line)))) + (collect-matching (=line-matches "^[*]") (make-peekstream :stream stream))))) + +(defun sources-that-failed-to-update () + (extract-failed-updates (latest-build-log-file))) diff --git a/upstream-git.lisp b/upstream-git.lisp index 3644494..1d12acf 100644 --- a/upstream-git.lisp +++ b/upstream-git.lisp @@ -45,6 +45,14 @@ (with-run-output (stream ("git" "rev-parse" "HEAD")) (read-line stream))))) +(defmethod source-cache-timestamp ((source git-source)) + (let ((unix-time-offset (load-time-value (encode-universal-time 0 0 0 1 1 1970 0)))) + (let ((checkout (ensure-source-cache source))) + (with-posix-cwd checkout + (with-run-output (stream ("git" "show" "-s" "--format=%ct")) + (let ((unix-time-string (read-line stream))) + (+ (parse-integer unix-time-string) unix-time-offset))))))) + (defmethod tag-data :around ((source branched-git-source)) (let ((tag (call-next-method)) (commit (commit-id source))) @@ -63,16 +71,63 @@ (:method ((source git-at-commit-source)) (format nil "~A" (commit source)))) +(defstruct (submodule (:type vector)) + name + path + sha1) + +(defun git-submodules (git-path) + (loop for line in + (run-output-lines "git" "-C" (truename git-path) "submodule" + "--quiet" + "foreach" + "--recursive" + "echo $name $sha1 $displaypath") + for (name sha1 path) = (split-spaces line) + collect (make-submodule :name (encode-string-for-filesystem name) + :path path + :sha1 sha1))) + +(defun full-git-archive (git-path target-ref prefix output-file) + "Create a tarball archive in OUTPUT-FILE of the full contents of the +git checkout GIT-PATH, including submodules. The repo is archived at +TARGET-REF, e.g. 'HEAD'. " + (run "git" "-C" (truename git-path) + "submodule" "update" "--init" "--recursive") + (let ((submodules (git-submodules git-path))) + (in-temporary-directory prefix + (let* ((temp-base *default-pathname-defaults*)) + (with-posix-cwd git-path + (run "git" "archive" + :format "tar" + :prefix (format nil "~A/" prefix) + "-o" (make-pathname :name (string-right-trim "/" prefix) + :type "tar" + :defaults temp-base) + target-ref) + (dolist (submodule submodules) + (with-posix-cwd (submodule-path submodule) + (let ((output (make-pathname :name (submodule-name submodule) + :type "tar" + :defaults temp-base))) + (run "git" "archive" + :format "tar" + "-o" output + :prefix (format nil "~A/~A/" + prefix (submodule-path submodule)) + (submodule-sha1 submodule)))))) + ;; Back in the temp directory + (dolist (tarball (directory "*.tar")) + (run "tar" "xf" tarball)) + (let ((combined "combined.tar") + (combined-tgz "combined.tar.gz")) + (run "tar" "cf" combined prefix) + (run "gzip" "-vn9" "-S" ".gz" combined) + (rename-file combined-tgz output-file)) + output-file)))) + (defmethod make-release-tarball ((source git-source) output-file) (let ((prefix (release-tarball-prefix source)) (checkout (ensure-source-cache source))) - (in-temporary-directory prefix - (let ((temptar (merge-pathnames "package.tar")) - (tempgz (merge-pathnames "package.tar.gz"))) - (with-posix-cwd checkout - (with-binary-run-output temptar - (run "git" "archive" :format "tar" :prefix prefix - (target-ref source))) - (run "gzip" "-vn9" temptar) - (copy tempgz output-file)))))) + (full-git-archive checkout (target-ref source) prefix output-file))) diff --git a/upstream-github.lisp b/upstream-github.lisp index b24c220..0633209 100644 --- a/upstream-github.lisp +++ b/upstream-github.lisp @@ -28,13 +28,15 @@ :tag (githappy:jref json '(0 "name")))))) -(defclass latest-github-release-source (http-source) +(defclass latest-github-release-source (git-source) ((release-url :initarg :release-url :accessor release-url) (release-tag :initarg :release-tag - :accessor release-tag))) + :accessor release-tag + :reader tag-data + :reader target-ref))) (defclass latest-github-tag-source (latest-github-release-source) ()) @@ -69,10 +71,7 @@ (defmethod release-tarball-prefix ((source latest-github-release-source)) (format nil "~A-~A/" (name source) (release-tag source))) -(defmethod create-source-cache ((source latest-github-release-source)) - (let ((cached (cache-object-file source))) - (ensure-directories-exist cached) - (curl (release-url source) cached) - (repack cached (release-tarball-prefix source) cached) - (probe-file cached))) - +(defmethod parse-location ((source latest-github-release-source) + location-string) + (setf (location source) location-string) + source) diff --git a/upstream-gitlab.lisp b/upstream-gitlab.lisp new file mode 100644 index 0000000..345f599 --- /dev/null +++ b/upstream-gitlab.lisp @@ -0,0 +1,38 @@ +;;;; upstream-gitlab.lisp +;;;; +;;;; Fetch some basic info about a source from gitlab +;;;; + +(in-package #:quicklisp-controller) + +(defun gitlab-project-id (url) + (ppcre:register-groups-bind (user project) + ("gitlab.com/(.*?)/(.*?)(.git)?$" url) + (when user + (format nil "~A%2F~A" user project)))) + +(defun gitlab-project-release-json (url) + (let* ((uri (format nil "https://gitlab.com/api/v4/projects/~A/releases" + (gitlab-project-id url))) + (request (make-instance 'githappy::request + :uri uri)) + (response (githappy::submit request)) + (json (yason:parse (githappy::utf8-string (githappy::body response))))) + json)) + +(defun gitlab-latest-tag-info (url) + (let ((json (gitlab-project-release-json url))) + (when json + (let ((tag-data (githappy:jref json '(0 "tag_name"))) + (sources (githappy:jref json '(0 "assets" "sources")))) + (dolist (source sources) + (when (equal (githappy:jref source "format") "tar.gz") + (return (list :tag tag-data + :url (githappy:jref source "url"))))))))) + + +(defclass latest-gitlab-release-source (latest-github-release-source) + ()) + +(defmethod github-info-plist ((source latest-gitlab-release-source)) + (gitlab-latest-tag-info (location source))) diff --git a/upstream-http.lisp b/upstream-http.lisp index f7e1990..35176a1 100644 --- a/upstream-http.lisp +++ b/upstream-http.lisp @@ -22,7 +22,11 @@ (merge-logical (format nil "~A/~A.dat" (project-name source) (string-digest (location source))) - "quicklisp-controller:http-cache;")) + "quicklisp-controller:upstream-cache;http;")) + +(defmethod source-cache-timestamp ((source http-source)) + (and (probe-file (cache-object-file source)) + (file-write-date (cache-object-file source)))) (defmethod make-release-tarball ((source http-source) output-file) (let ((prefix (release-tarball-prefix source)) @@ -77,7 +81,7 @@ (with-binary-run-output "temp.dat" (run "bunzip2" "-c" "temp.bz2")) (run "gzip" "temp.dat") - (rename-file "temp.dat.gz" cached)) + (alexandria:copy-file "temp.dat.gz" cached)) (probe-file cached))) (defmethod update-source-cache ((source http-source)) diff --git a/upstream-vcs.lisp b/upstream-vcs.lisp index c4e53aa..630d36c 100644 --- a/upstream-vcs.lisp +++ b/upstream-vcs.lisp @@ -79,7 +79,7 @@ (merge-logical (format nil "~A/~A/" (project-name source) (string-digest (location source))) - "quicklisp-controller:vcs-cache;"))) + "quicklisp-controller:upstream-cache;vcs;"))) (defmethod ensure-source-cache ((source vcs-source)) (let ((pathname (cached-checkout-directory source))) @@ -96,6 +96,8 @@ (probe-file pathname))) + + ;;; Tags of some sort (defclass tagged-mixin () diff --git a/upstream.lisp b/upstream.lisp index c75a0a1..cb517ae 100644 --- a/upstream.lisp +++ b/upstream.lisp @@ -26,6 +26,12 @@ (defmethod base-directory ((source upstream-source)) (pathname (directory-namestring (source-file source)))) +(defgeneric days-old (source) + (:method (source) + (truncate (- (get-universal-time) + (file-write-date (source-file source))) + 86400))) + (defgeneric print-source (source stream) (:method (source stream) (format stream "~S ~S" @@ -78,6 +84,25 @@ (:documentation "Try to update cached data for SOURCE.")) +(defgeneric source-cache-timestamp (source) + (:documentation "Return the timestamp, as a universal time, at which + the source cache was last updated by the upstream. Return NIL if the + timestamp can't be determined.") + (:method ((source t)) + nil)) + +(defgeneric source-cache-age-or-nil (source) + (:documentation "Return the age, in days, of the source's cached + data. For VCS, it is the age of commit. For HTTP/HTTPS, it's the + date on the last-modified. If the age can't be determined, return + NIL.") + (:method (source) + (let ((time (source-cache-timestamp source))) + (and time + (/ (- (get-universal-time) (source-cache-timestamp source)) + 86400.0))))) + + ;;; ;;; Loading & mapping sources from the quicklisp-projects directory. ;;; @@ -146,8 +171,8 @@ (defun collect-sources-if (fun) (let ((result '())) (map-sources (lambda (source) - (when (funcall fun source) - (push source result)))) + (when (funcall fun source) + (push source result)))) (sort result 'string< :key 'name))) (defun pmap-sources (fun &key (parallel-key #'source-host) @@ -187,7 +212,7 @@ (not (probe-file (make-pathname :type nil :name "fresh-cache" - :defaults (project-name-source-file (name source)) ))))) + :defaults (project-name-source-file (name source)) ))))) (defun find-source (project-name) (let* ((name (string-downcase project-name)) @@ -225,13 +250,13 @@ (defun missing-commands () (let ((missing '()) - (tried (make-string-table))) + (tried (make-string-table))) (map-sources (lambda (source) (let ((command (command source))) - (when command - (unless (gethash command tried) - (setf (gethash command tried) t) - (unless (ignore-errors (run command "--version")) - (push command missing))))))) + (when command + (unless (gethash command tried) + (setf (gethash command tried) t) + (unless (ignore-errors (run command "--version")) + (push command missing))))))) missing)) diff --git a/utils.lisp b/utils.lisp index 2579820..50fb8b1 100644 --- a/utils.lisp +++ b/utils.lisp @@ -42,7 +42,7 @@ (defun rm-rf (path) (unless *rm-rf-debug* - (run "rm" "-rf" (native path)))) + (run "rm" "-rf" (native (translate-logical-pathname path))))) (defun call-in-temporary-directory (template-pathname fun) (flet ((random-temporary () @@ -176,19 +176,21 @@ template pathname." (save-forms (list form) file)) (defun tarball-contents (file) - (with-run-output (stream ("tar" "tzf" (native file))) - (loop for line = (read-line stream nil) - while line collect line))) + (run-output-lines "tar" "tzf" file)) (defun tarball-prefix (file) + "For a tarball that unpacks into a subdirectory (e.g. 'foo/foo.asd', +'foo/package.asd', etc), extract the subdirectory string. Errors if +the subdirectory is absent or inconsistent." (let ((contents (tarball-contents file))) - (let ((prefix (subseq (first contents) 0 - (1+ (position #\/ (first contents)))))) - (dolist (entry contents prefix) - (unless (and (<= (length prefix) (length entry)) - (string= prefix entry :end2 (length prefix))) - (error "Tarball ~A lacks consistent prefix output directory" - file)))))) + (let ((prefixes (loop for path in contents + collect + (subseq path 0 (position #\/ path))))) + (unless (every 'string= prefixes (rest prefixes)) + (error "Inconsistent prefixes in ~A: ~S" + file + (remove-duplicates prefixes :test 'string=))) + (format nil "~A/" (first prefixes))))) (defun tarball-canonical-name (file) (string-right-trim "/" (tarball-prefix file))) @@ -288,3 +290,42 @@ template pathname." (#\Nul (write-string "[nul]" s)) (t (write-char char s)))))) + +(defun gist-file (&key (description "No description") pathname (public t)) + (unless pathname + (error "Pathname required")) + (flet ((js (&rest args) + (apply #'githappy:js args)) + (table (&rest args) + (apply #'githappy:table args))) + (let ((value (alexandria:read-file-into-string pathname)) + (key (file-namestring pathname))) + (githappy:create-gist :body + (githappy:js "description" description + "public" public + "files" (table key (table "content" value))))))) + + +(defun gist-string (&key (description "No description") string (public t)) + (unless string + (error "string required")) + (flet ((js (&rest args) + (apply #'githappy:js args)) + (table (&rest args) + (apply #'githappy:table args))) + (let ((value string) + (key "text.md")) + (githappy:create-gist :body + (githappy:js "description" description + "public" public + "files" (table key (table "content" value))))))) + +(defun how-long-ago (days) + (cond ((< days 1) + "less than a day") + ((< days 21) + (format nil "~:D day~:P" (truncate days))) + ((< days 365) + (format nil "~:D week~:P" (truncate days 7.0))) + (t + (format nil "~:D year~:P" (truncate days 365.0)))))