update with upstream's testdriver/onboarding-test branch; also add self hosting

Author: chottuthejimmy

.github/workflows/self-host.yml

@@ -0,0 +1,76 @@
+name: TestDriver - Self Hosted
+
+on:
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  test:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        test_file:
+          - testdriver/installation/programmatic_installation_existing_user.yaml
+          - testdriver/installation/desktop_app_installation_new_user_exe.yaml
+          - testdriver/installation/desktop_app_installation_new_user_msix.yaml
+          - testdriver/installation/desktop_app_installation_new_user_winget.yaml
+
+    steps:
+      - name: Checkout the repo to access the test files
+        uses: actions/checkout@v4
+        
+      - name: Setup AWS Instance
+        id: aws-setup
+        run: |
+          OUTPUT=$(./setup/aws/spawn-runner.sh | tee /dev/stderr) # Capture and display output
+          echo "$OUTPUT"
+          PUBLIC_IP=$(echo "$OUTPUT" | grep "PUBLIC_IP=" | cut -d'=' -f2)
+          INSTANCE_ID=$(echo "$OUTPUT" | grep "INSTANCE_ID=" | cut -d'=' -f2)
+          AWS_REGION=$(echo "$OUTPUT" | grep "AWS_REGION=" | cut -d'=' -f2)
+          echo "public-ip=$PUBLIC_IP" >> $GITHUB_OUTPUT
+          echo "instance-id=$INSTANCE_ID" >> $GITHUB_OUTPUT
+          echo "aws-region=$AWS_REGION" >> $GITHUB_OUTPUT
+        env:
+          FORCE_COLOR: 3
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: us-east-2
+          AWS_LAUNCH_TEMPLATE_ID: lt-00d02f31cfc602f27
+          AMI_ID: ami-055cd47506a2f39bb
+          RESOLUTION_WIDTH: 1920
+          RESOLUTION_HEIGHT: 1080
+      - name: Run TestDriver
+        run: npx testdriverai@latest run ${{ matrix.test_file }} --ip="${{ steps.aws-setup.outputs.public-ip }}"
+        env:
+          TD_API_KEY: ${{ secrets.TD_API_KEY }}
+          TD_WEBSITE: https://testdriver-sandbox.vercel.app
+          TD_THIS_FILE: ${{ matrix.test_file }}
+
+      - name: Upload TestDriver AI CLI logs
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: testdriverai-cli-logs-${{ matrix.test_file }}
+          path: /tmp/testdriverai-cli-*.log
+          if-no-files-found: warn
+          retention-days: 30
+
+      - name: Upload test results as artifact
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-results-${{ matrix.test_file }}
+          path: out.xml
+          retention-days: 30
+          
+      - name: Shutdown AWS Instance
+        if: always()
+        run: aws ec2 terminate-instances --region "$AWS_REGION" --instance-ids "$INSTANCE_ID"
+        env:
+          AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+          AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
+          AWS_REGION: ${{ steps.aws-setup.outputs.aws-region }}
+          INSTANCE_ID: ${{ steps.aws-setup.outputs.instance-id }}

.github/workflows/testdriver.yml .github/workflows/testdriver_v5.yml.github/workflows/testdriver.yml renamed to .github/workflows/testdriver_v5.yml

@@ -1,23 +1,26 @@
-name: TestDriver v6
+name: TestDriver - Installation Tests
 
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   test-workflow:
     runs-on: ubuntu-latest
     strategy:
+      fail-fast: false
       matrix:
         test_file:
-          - testdriver/_focus.yaml
-          - testdriver/install_pos/install_pos_exe.yaml
+          - testdriver/onboarding/onboarding.yaml
     steps:
       - name: Checkout the repo to access the test files
         uses: actions/checkout@v4
 
-      - name: Run test
+      - name: Run TestDriverAI
         run: |
-          npx testdriverai@beta run ${{ matrix.test_file }} --sandbox-instance=c5.2xlarge
+          npx testdriverai@latest run ${{ matrix.test_file }} --sandbox-instance=c5.2xlarge
         env:
           FORCE_COLOR: 3
           TD_API_KEY: ${{ secrets.TESTDRIVER_API_KEY }}

.github/workflows/testdriver_v6.yml

@@ -5,4 +5,4 @@ steps:
     commands:
       - command: exec
         lang: pwsh
-        code: dashcam -t '${TD_THIS_FILE}' -p
+        code: dashcam -t '${TD_THIS_FILE}' -p

testdriver/lifecycle/postrun.yaml

@@ -8,4 +8,4 @@ steps:
         code: dashcam track --name=TestDriver --type=application --pattern="C:\Users\testdriver\Documents\testdriver.log"
       - command: exec
         lang: pwsh
-        code: dashcam start
+        code: dashcam start

testdriver/lifecycle/prerun.yaml

@@ -1,7 +1,7 @@
 version: 6.0.11
 session: 67f00511acbd9ccac373edf7
 steps:
-  - prompt: install pfd
+  - prompt: Install Desktop App
     commands:
       - command: exec
         lang: pwsh

testdriver/lifecycle/provision.yaml

@@ -0,0 +1,73 @@
+version: 6.0.11
+steps:
+  - prompt: 
+      Continue following orders to finish the LTM onboarding flow
+    commands: 
+      - command: wait
+        timeout: 10000
+      - command: hover-text
+        text: I'm Ready
+        description: A dark button in the bottom left named 'I'm Ready'
+        action: click
+      - command: wait
+        timeout: 7000 
+      - command: hover-image 
+        description: click on the Source Control button (settings icon with dots) selected with blue shadow, to the right of the 'LTM-2' button
+        action: click       
+
+  - prompt: Open Power Menu via Home Icon (hold Ctrl then click Enter), click Continue, then go to Pieces Copilot and ensure it opens. Click GPT-4.1, then Cloud, press Escape, and select the prompt; 'Generate a stand-up styled summary of my recent work'. Assert that Copilot responds after prompt is sent.
+    commands:
+      - command: press-keys
+        keys:
+          - ctrlleft
+          - enter
+      - command: assert
+        expect: A pop-up menu is displayed  
+      - command: hover-text
+        text: Continue
+        description: A dark button named 'Continue'
+        action: click
+      - command: hover-text
+        text: Pieces Copilot
+        description: click on the text 'Pieces Copilot' in the horizontal menu bar    
+        action: click
+      - command: wait
+        timeout: 5000  
+      - command: assert
+        expect: A text 'New chat' is displayed
+      - command: hover-text
+        text: GPT-4.1
+        description: A button in the bottom left that has 'OpenAI/Chatgpt' logo named 'GPT-4.1'
+        action: click
+      - command: wait
+        timeout: 5000  
+      - command: hover-text
+        text: Cloud
+        description: a button named 'Cloud' to access Cloud LLMs
+        action: click
+      - command: wait
+        timeout: 5000          
+      - command: press-keys
+        keys: [escape]
+      - command: hover-text
+        text: Generate a stand-up styled summary of my recent work.
+        description: A suggested prompt
+        action: click
+      - command: wait
+        timeout: 75000  
+      - command: assert
+        expect: Copilot Responds to the message sent
+      - command: press-keys
+        keys:
+          - ctrlleft
+          - enter
+      - command: assert
+        expect: A pop-up menu is displayed 
+      - command: hover-text
+        text: Pieces Drive
+        description: click on the text 'Pieces Drive' in the horizontal menu bar    
+        action: click
+      - command: wait
+        timeout: 5000  
+      - command: assert
+        expect: A text 'Saved Materials' is displayed