diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index cd3f6969..45af68d2 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -1,19 +1,145 @@ name: Test on: + schedule: + - cron: "0 11 * * *" # 11am UTC = 3`am PST + + pull_request: + types: [ labeled, closed ] + workflow_dispatch: inputs: - unity_versions: - description: 'Unity version (value: 2018, 2019, 2020. separated by commas)' + unity_version: + description: 'Unity version (value: 2018, 2019, 2020)' default: '2019' required: true + include_test_types: + description: 'Specify the only types of tests to run, separated by comma. See TestTypesEnum in build.gradle for options.' + default: '' + required: false + exclude_test_types: + description: 'Specify the types of tests to exclude, separated by comma. See TestTypesEnum in build.gradle for options.' + default: '' + required: false + include_test_modules: + description: 'Specify the only modules to test against, separated by comma. See TestModulesEnum in build.gradle for options.' + default: '' + required: false + exclude_test_modules: + description: 'Specify the modules to exclude from testing against, separated by comma. See TestModulesEnum in build.gradle for options.' + default: '' + required: false + exclude_tests: + description: 'Specify the tests to exclude, separated by comma. See the tasks in build.gradle for options.' + default: '' + required: false env: pythonVersion: '3.7' artifactRetentionDays: 2 jobs: + check_and_prepare: + runs-on: ubuntu-latest + outputs: + unity_version: ${{ steps.set_outputs.outputs.unity_version }} + include_test_types: ${{ steps.set_outputs.outputs.include_test_types }} + exclude_test_types: ${{ steps.set_outputs.outputs.exclude_test_types }} + include_test_modules: ${{ steps.set_outputs.outputs.include_test_modules }} + exclude_test_modules: ${{ steps.set_outputs.outputs.exclude_test_modules }} + exclude_tests: ${{ steps.set_outputs.outputs.exclude_tests }} + steps: + - id: set_outputs + run: | + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "unity_version=${{ github.event.inputs.unity_version }}" >> $GITHUB_OUTPUT + echo "include_test_types=${{ github.event.inputs.include_test_types }}" >> $GITHUB_OUTPUT + echo "exclude_test_types=${{ github.event.inputs.exclude_test_types }}" >> $GITHUB_OUTPUT + echo "include_test_modules=${{ github.event.inputs.include_test_modules }}" >> $GITHUB_OUTPUT + echo "exclude_test_modules=${{ github.event.inputs.exclude_test_modules }}" >> $GITHUB_OUTPUT + echo "exclude_tests=${{ github.event.inputs.exclude_tests }}" >> $GITHUB_OUTPUT + else + # inputs are not available for non "workflow_dispatch" events. Therefore, set default value here. + echo "unity_version=2019" >> $GITHUB_OUTPUT + echo "include_test_types=" >> $GITHUB_OUTPUT + echo "exclude_test_types=" >> $GITHUB_OUTPUT + echo "include_test_modules=" >> $GITHUB_OUTPUT + echo "exclude_test_modules=" >> $GITHUB_OUTPUT + echo "exclude_tests=" >> $GITHUB_OUTPUT + fi + + - name: Print output + run: | + echo outputs.unity_version : ${{ steps.set_outputs.outputs.unity_version }} + echo outputs.include_test_types : ${{ steps.set_outputs.outputs.include_test_types }} + echo outputs.exclude_test_types : ${{ steps.set_outputs.outputs.exclude_test_types }} + echo outputs.include_test_modules : ${{ steps.set_outputs.outputs.include_test_modules }} + echo outputs.exclude_test_modules : ${{ steps.set_outputs.outputs.exclude_test_modules }} + echo outputs.exclude_tests : ${{ steps.set_outputs.outputs.exclude_tests }} + test_on_macos: + name: test-macOS-unity${{ needs.check_and_prepare.outputs.unity_version }} runs-on: macos-latest + needs: [check_and_prepare] + strategy: + fail-fast: false steps: - - run: echo "Hello World" + - uses: actions/checkout@v3 + - id: build_setup + uses: ./gha/build_setup + timeout-minutes: 30 + with: + unity_version: ${{ needs.check_and_prepare.outputs.unity_version }} + platform: macOS + python_version: ${{ env.pythonVersion }} + unity_username: ${{ secrets.UNITY_USERNAME }} + unity_password: ${{ secrets.UNITY_PASSWORD }} + unity_serial_id: ${{ secrets.SERIAL_ID }} + + - name: Set Unity Env for EDM4U build script + shell: bash + run: echo "UNITY_EXE=${{ env.UNITY_ROOT_DIR }}/Unity.app/Contents/MacOS/Unity" >> $GITHUB_ENV + + - name: Run tests + shell: bash + timeout-minutes: 60 + run: | + # DISABLE: NUnit test is currently broken from Unity 2019 + # DISABLE: testVersionHandlerActivationBatchMode is causing timeout. + ./gradlew test -q \ + -PINTERACTIVE_MODE_TESTS_ENABLED=0 \ + -PINCLUDE_TEST_TYPES="${{ needs.check_and_prepare.outputs.include_test_types }}" \ + -PEXCLUDE_TEST_TYPES="NUnit,${{ needs.check_and_prepare.outputs.exclude_test_types }}" \ + -PINCLUDE_TEST_MODULES="${{ needs.check_and_prepare.outputs.include_test_modules }}" \ + -PEXCLUDE_TEST_MODULES="${{ needs.check_and_prepare.outputs.exclude_test_modules }}" \ + -PEXCLUDE_TESTS="testVersionHandlerActivationBatchMode,${{ needs.check_and_prepare.outputs.exclude_tests }}" + + - name: Print test log + if: always() + shell: bash + continue-on-error: true + run: cat test_output/test*IntegrationTestsBatchMode/*.log + + - name: Obtain Failed tests + if: always() + shell: bash + continue-on-error: true + run: | + cat test_output/test*/*_test.log | grep "^Test .* PASSED$" + cat test_output/test*/*_test.log | grep "^Test .* FAILED$" + cat test_output/test*/*_test.log | grep "^Test .* SKIPPED$" + + - name: Return Unity license + if: always() + uses: firebase/firebase-unity-sdk/gha/unity@main + with: + version: ${{ inputs.unity_version }} + release_license: "true" + + - name: Upload build logs + uses: actions/upload-artifact@v3 + if: ${{ !cancelled() }} + with: + name: logs + path: test_output/test*/*.log + retention-days: ${{ env.artifactRetentionDays }} diff --git a/build.gradle b/build.gradle index 7acdf907..28001511 100644 --- a/build.gradle +++ b/build.gradle @@ -526,6 +526,9 @@ void setTestProperties(Task task, TestTypeEnum type, TestModuleEnum module) { */ task(testTestTypeEnum) { task -> setTestProperties(task, TestTypeEnum.GRADLE, TestModuleEnum.TOOL) + doFirst { + ReportTestStarted(task) + } doLast { def expectedTestTypeEnumCompleteSet = new HashSet([ @@ -570,6 +573,9 @@ task(testTestTypeEnum) { task -> */ task(testTestModuleEnum) { task -> setTestProperties(task, TestTypeEnum.GRADLE, TestModuleEnum.TOOL) + doFirst { + ReportTestStarted(task) + } doLast { def expectedTestModuleEnumCompleteSet = new HashSet([ @@ -1480,6 +1486,9 @@ Task createUnityTestTask(String taskName, String description, doLast { EvaluateTestResult(task) } + doFirst { + ReportTestStarted(task) + } setTestProperties(task, testType, testModule) } @@ -1713,54 +1722,62 @@ task testDownloadArtifacts(type: GradleBuild) { task -> doLast { EvaluateTestResult(task) } + doFirst { + ReportTestStarted(task) + } finalizedBy "reportAllTestsResult" } + +/* + * Report when a test starts to run + * + * @param testTask Task for test to start. + */ +void ReportTestStarted(Task testTask) { + println sprintf("Test %s STARTED", testTask.name) +} + /* * Evaluate previously-ran test result * * @param testTask Task for previously-ran test */ void EvaluateTestResult(Task testTask) { + Boolean succeeded = false if (testTask.class.simpleName.startsWith("Exec")) { - if (testTask.execResult.exitValue != 0) { - String errorMsg = sprintf("Test %s FAILED", testTask.name) - println sprintf("::error::%s", errorMsg) - project.ext.testSessions.add(new TestSession( - name: testTask.name, - type: testTask.ext.testType, - module: testTask.ext.testModule, - isPassed: false)) - if (!project.ext.continueOnFailForTestsEnabled) { - throw new GradleException(errorMsg) - } - } else { - println sprintf("::debug::Test %s PASSED", testTask.name, testTask.execResult.exitValue) - project.ext.testSessions.add(new TestSession( - name: testTask.name, - type: testTask.ext.testType, - module: testTask.ext.testModule, - isPassed: true)) + if (testTask.execResult.exitValue == 0) { + succeeded = true } } else if (testTask.class.simpleName.startsWith("DefaultTask") || testTask.class.simpleName.startsWith("GradleBuild")) { if (testTask.state.didWork && testTask.state.failure == null) { - project.ext.testSessions.add(new TestSession( - name: testTask.name, - type: testTask.ext.testType, - module: testTask.ext.testModule, - isPassed: true)) - } else { - project.ext.testSessions.add(new TestSession( - name: testTask.name, - type: testTask.ext.testType, - module: testTask.ext.testModule, - isPassed: false)) + succeeded = true } } else { throw new GradleException( sprintf("Unsupported test class %s", testTask.class.simpleName)) } + + if (succeeded) { + println sprintf("Test %s PASSED", testTask.name) + project.ext.testSessions.add(new TestSession( + name: testTask.name, + type: testTask.ext.testType, + module: testTask.ext.testModule, + isPassed: true)) + } else { + String errorMsg = sprintf("Test %s FAILED", testTask.name) + println sprintf("::error::%s", errorMsg) + project.ext.testSessions.add(new TestSession( + name: testTask.name, + type: testTask.ext.testType, + module: testTask.ext.testModule, + isPassed: false)) + if (!project.ext.continueOnFailForTestsEnabled) { + throw new GradleException(errorMsg) + } + } } Task reportAllTestsResult = tasks.create ( @@ -1769,26 +1786,36 @@ Task reportAllTestsResult = tasks.create ( type: Task ).with { doLast { + if (project.ext.testSessions.isEmpty()) { + return + } + + println "\n\n[Test Summary]" int failedCount = 0 int totalCount = 0 project.ext.testSessions.each { session -> String resultStr ++totalCount + String logType = "" if (session.isPassed) { resultStr = "PASSED" } else { resultStr = "FAILED" + logType = "::error::" ++failedCount } - println sprintf("Test %s %s [%s/%s]", + println sprintf("%sTest %s %s [%s/%s]", + logType, session.name, resultStr, session.type, session.module) } + println "--------------------------------------" + println sprintf("::notice::%s out of %d test(s) passed", totalCount - failedCount, totalCount) if (failedCount > 0) { throw new GradleException( - sprintf("%d out of %d tests failed", failedCount, totalCount)) + sprintf("%d out of %d test(s) failed", failedCount, totalCount)) } } } @@ -1805,6 +1832,9 @@ Task testPackageUploader = createPythonTask( doLast { EvaluateTestResult(task) } + doFirst { + ReportTestStarted(task) + } setTestProperties(task, TestTypeEnum.PYTHON, TestModuleEnum.TOOL) } @@ -1821,7 +1851,10 @@ createPythonTask( doLast { EvaluateTestResult(task) } - } + doFirst { + ReportTestStarted(task) + } +} createPythonTask( "testGenGuids", @@ -1836,6 +1869,9 @@ createPythonTask( doLast { EvaluateTestResult(task) } + doFirst { + ReportTestStarted(task) + } } createPythonTask( @@ -1851,6 +1887,9 @@ createPythonTask( doLast { EvaluateTestResult(task) } + doFirst { + ReportTestStarted(task) + } } task updateEmbeddedGradleWrapper(type: Zip) { diff --git a/gha/build_setup/action.yml b/gha/build_setup/action.yml index 59109464..aaacd456 100644 --- a/gha/build_setup/action.yml +++ b/gha/build_setup/action.yml @@ -65,7 +65,7 @@ runs: with: version: ${{ inputs.unity_version }} # iOS build support is always required to build EDM4U - platforms: "${{ inputs.platform }},iOS" + platforms: "${{ inputs.platform }},iOS,Android" username: ${{ inputs.unity_username }} password: ${{ inputs.unity_password }} serial_ids: ${{ inputs.unity_serial_id }}