Merge pull request #2641 from umami-software/analytics

v2.11.0

Author: mikecao

.gitignore

@@ -22,6 +22,7 @@ node_modules
 # misc
 .DS_Store
 .idea
+.yarn
 *.iml
 *.log
 .vscode

cypress.config.ts

@@ -4,4 +4,9 @@ export default defineConfig({
   e2e: {
     baseUrl: 'http://localhost:3000',
   },
+  // default username / password on init
+  env: {
+    umami_user: 'admin',
+    umami_password: 'umami',
+  },
 });

cypress/docker-compose.yml

@@ -43,9 +43,10 @@ services:
       - CYPRESS_umami_user=admin
       - CYPRESS_umami_password=umami
     volumes:
-      - ../tsconfig.json:/tsconfig.json
+      - ./tsconfig.json:/tsconfig.json
       - ../cypress.config.ts:/cypress.config.ts
       - ./:/cypress
       - ../node_modules/:/node_modules
+      - ../src/lib/crypto.ts:/src/lib/crypto.ts
 volumes:
   umami-db-data:

cypress/e2e/login.cy.ts

@@ -6,8 +6,12 @@ describe('Login tests', () => {
     },
     () => {
       cy.visit('/login');
-      cy.getDataTest('input-username').find('input').type(Cypress.env('umami_user'));
-      cy.getDataTest('input-password').find('input').type(Cypress.env('umami_password'));
+      cy.getDataTest('input-username').find('input').click();
+      cy.getDataTest('input-username').find('input').type(Cypress.env('umami_user'), { delay: 50 });
+      cy.getDataTest('input-password').find('input').click();
+      cy.getDataTest('input-password')
+        .find('input')
+        .type(Cypress.env('umami_password'), { delay: 50 });
       cy.getDataTest('button-submit').click();
       cy.url().should('eq', Cypress.config().baseUrl + '/dashboard');
       cy.getDataTest('button-profile').click();

cypress/e2e/website.cy.ts

@@ -10,8 +10,10 @@ describe('Website tests', () => {
     cy.visit('/settings/websites');
     cy.getDataTest('button-website-add').click();
     cy.contains(/Add website/i).should('be.visible');
-    cy.getDataTest('input-name').find('input').wait(500).type('Add test', { delay: 50 });
-    cy.getDataTest('input-domain').find('input').wait(500).type('addtest.com', { delay: 50 });
+    cy.getDataTest('input-name').find('input').click();
+    cy.getDataTest('input-name').find('input').type('Add test', { delay: 50 });
+    cy.getDataTest('input-domain').find('input').click();
+    cy.getDataTest('input-domain').find('input').type('addtest.com', { delay: 50 });
     cy.getDataTest('button-submit').click();
     cy.get('td[label="Name"]').should('contain.text', 'Add test');
     cy.get('td[label="Domain"]').should('contain.text', 'addtest.com');
@@ -26,27 +28,23 @@ describe('Website tests', () => {
         cy.deleteWebsite(websiteId);
       });
     cy.visit('/settings/websites');
-    cy.contains('Add test').should('not.exist');
+    cy.contains(/Add test/i).should('not.exist');
   });
 
-  it.only('Edit a website', () => {
+  it('Edit a website', () => {
     // prep data
     cy.addWebsite('Update test', 'updatetest.com');
     cy.visit('/settings/websites');
 
     // edit website
     cy.getDataTest('link-button-edit').first().click();
     cy.contains(/Details/i).should('be.visible');
-    cy.getDataTest('input-name')
-      .find('input')
-      .wait(500)
-      .clear()
-      .type('Updated website', { delay: 50 });
-    cy.getDataTest('input-domain')
-      .find('input')
-      .wait(500)
-      .clear()
-      .type('updatedwebsite.com', { delay: 50 });
+    cy.getDataTest('input-name').find('input').click();
+    cy.getDataTest('input-name').find('input').clear();
+    cy.getDataTest('input-name').find('input').type('Updated website', { delay: 50 });
+    cy.getDataTest('input-domain').find('input').click();
+    cy.getDataTest('input-domain').find('input').clear();
+    cy.getDataTest('input-domain').find('input').type('updatedwebsite.com', { delay: 50 });
     cy.getDataTest('button-submit').click({ force: true });
     cy.getDataTest('input-name').find('input').should('have.value', 'Updated website');
     cy.getDataTest('input-domain').find('input').should('have.value', 'updatedwebsite.com');
@@ -69,7 +67,7 @@ describe('Website tests', () => {
         cy.deleteWebsite(websiteId);
       });
     cy.visit('/settings/websites');
-    cy.contains('Add test').should('not.exist');
+    cy.contains(/Add test/i).should('not.exist');
   });
 
   it('Delete a website', () => {
@@ -86,6 +84,6 @@ describe('Website tests', () => {
     cy.contains(/Type DELETE in the box below to confirm./i).should('be.visible');
     cy.get('input[name="confirm"').type('DELETE');
     cy.get('button[type="submit"]').click();
-    cy.contains('Delete test').should('not.exist');
+    cy.contains(/Delete test/i).should('not.exist');
   });
 });

db/clickhouse/migrations/02_add_visit_id.sql

@@ -0,0 +1,90 @@
+CREATE TABLE umami.website_event_join
+(
+    session_id UUID,
+    visit_id UUID,
+    created_at DateTime('UTC')
+)
+    engine = MergeTree
+        ORDER BY (session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_join
+SELECT DISTINCT
+    s.session_id,
+    generateUUIDv4() visit_id,
+    s.created_at
+FROM (SELECT DISTINCT session_id,
+        date_trunc('hour', created_at) created_at
+    FROM website_event) s;
+
+-- create new table
+CREATE TABLE umami.website_event_new
+(
+    website_id UUID,
+    session_id UUID,
+    visit_id UUID,
+    event_id UUID,
+    hostname LowCardinality(String),
+    browser LowCardinality(String),
+    os LowCardinality(String),
+    device LowCardinality(String),
+    screen LowCardinality(String),
+    language LowCardinality(String),
+    country LowCardinality(String),
+    subdivision1 LowCardinality(String),
+    subdivision2 LowCardinality(String),
+    city String,
+    url_path String,
+    url_query String,
+    referrer_path String,
+    referrer_query String,
+    referrer_domain String,
+    page_title String,
+    event_type UInt32,
+    event_name String,
+    created_at DateTime('UTC'),
+    job_id UUID
+)
+    engine = MergeTree
+        ORDER BY (website_id, session_id, created_at)
+        SETTINGS index_granularity = 8192;
+
+INSERT INTO umami.website_event_new
+SELECT we.website_id,
+    we.session_id,
+    j.visit_id,
+    we.event_id,
+    we.hostname,
+    we.browser,
+    we.os,
+    we.device,
+    we.screen,
+    we.language,
+    we.country,
+    we.subdivision1,
+    we.subdivision2,
+    we.city,
+    we.url_path,
+    we.url_query,
+    we.referrer_path,
+    we.referrer_query,
+    we.referrer_domain,
+    we.page_title,
+    we.event_type,
+    we.event_name,
+    we.created_at,
+    we.job_id
+FROM umami.website_event we
+JOIN umami.website_event_join j
+    ON we.session_id = j.session_id
+        and date_trunc('hour', we.created_at) = j.created_at
+
+RENAME TABLE umami.website_event TO umami.website_event_old;
+RENAME TABLE umami.website_event_new TO umami.website_event;
+
+/*
+
+ DROP TABLE umami.website_event_old
+ DROP TABLE umami.website_event_join
+
+ */

db/clickhouse/schema.sql

@@ -3,6 +3,7 @@ CREATE TABLE umami.website_event
 (
     website_id UUID,
     session_id UUID,
+    visit_id UUID,
     event_id UUID,
     --sessions
     hostname LowCardinality(String),

db/mysql/migrations/05_add_visit_id/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE `website_event` ADD COLUMN `visit_id` VARCHAR(36) NULL;
+
+UPDATE `website_event` we
+JOIN (SELECT DISTINCT
+        s.session_id,
+        s.visit_time,
+        BIN_TO_UUID(RANDOM_BYTES(16) & 0xffffffffffff0fff3fffffffffffffff | 0x00000000000040008000000000000000) uuid
+    FROM (SELECT DISTINCT session_id,
+            DATE_FORMAT(created_at, '%Y-%m-%d %H:00:00') visit_time
+        FROM `website_event`) s) a
+    ON we.session_id = a.session_id and DATE_FORMAT(we.created_at, '%Y-%m-%d %H:00:00') = a.visit_time
+SET we.visit_id = a.uuid
+WHERE we.visit_id IS NULL;
+
+ALTER TABLE `website_event` MODIFY `visit_id` VARCHAR(36) NOT NULL;
+
+-- CreateIndex
+CREATE INDEX `website_event_visit_id_idx` ON `website_event`(`visit_id`);
+
+-- CreateIndex
+CREATE INDEX `website_event_website_id_visit_id_created_at_idx` ON `website_event`(`website_id`, `visit_id`, `created_at`);

db/mysql/schema.prisma

@@ -92,6 +92,7 @@ model WebsiteEvent {
   id             String    @id() @map("event_id") @db.VarChar(36)
   websiteId      String    @map("website_id") @db.VarChar(36)
   sessionId      String    @map("session_id") @db.VarChar(36)
+  visitId        String    @map("visit_id") @db.VarChar(36)
   createdAt      DateTime? @default(now()) @map("created_at") @db.Timestamp(0)
   urlPath        String    @map("url_path") @db.VarChar(500)
   urlQuery       String?   @map("url_query") @db.VarChar(500)
@@ -107,6 +108,7 @@ model WebsiteEvent {
 
   @@index([createdAt])
   @@index([sessionId])
+  @@index([visitId])
   @@index([websiteId])
   @@index([websiteId, createdAt])
   @@index([websiteId, createdAt, urlPath])
@@ -115,6 +117,7 @@ model WebsiteEvent {
   @@index([websiteId, createdAt, pageTitle])
   @@index([websiteId, createdAt, eventName])
   @@index([websiteId, sessionId, createdAt])
+  @@index([websiteId, visitId, createdAt])
   @@map("website_event")
 }
 

db/postgresql/migrations/05_add_visit_id/migration.sql

@@ -0,0 +1,22 @@
+-- AlterTable
+ALTER TABLE "website_event" ADD COLUMN "visit_id" UUID NULL;
+
+UPDATE "website_event" we
+SET visit_id = a.uuid
+FROM (SELECT DISTINCT
+        s.session_id,
+        s.visit_time,
+        gen_random_uuid() uuid
+    FROM (SELECT DISTINCT session_id,
+            date_trunc('hour', created_at) visit_time
+        FROM "website_event") s) a
+WHERE we.session_id = a.session_id 
+    and date_trunc('hour', we.created_at) = a.visit_time;
+
+ALTER TABLE "website_event" ALTER COLUMN "visit_id" SET NOT NULL;
+
+-- CreateIndex
+CREATE INDEX "website_event_visit_id_idx" ON "website_event"("visit_id");
+
+-- CreateIndex
+CREATE INDEX "website_event_website_id_visit_id_created_at_idx" ON "website_event"("website_id", "visit_id", "created_at");