Skip to content

Commit c4ad2cb

Browse files
warren830warren830
authored
feat(q-dev): enrich logging fields, separate dashboards, add E2E tests (#8786)
* feat(q-dev): enrich logging fields, separate dashboards by data source, add E2E tests - Add new fields to chat_log: CodeReferenceCount, WebLinkCount, HasFollowupPrompts (from codeReferenceEvents, supplementaryWebLinksEvent, followupPrompts in JSON) - Add new fields to completion_log: LeftContextLength, RightContextLength (from leftContext/rightContext in JSON) - Update s3_logging_extractor to parse and populate new fields - Add migration script 20260319_add_logging_fields - Create qdev_feature_metrics dashboard for legacy by_user_analytic data - Reorganize qdev_executive dashboard with Row dividers labeling data sources and cross-dashboard navigation links - Enrich qdev_logging dashboard with new panels: Chat Trigger Type Distribution, Response Enrichment Breakdown, Completion Context Size Trends, Response Enrichment Trends - Fix SQL compatibility with only_full_group_by mode in executive dashboard (Weekly Active Users Trend, New vs Returning Users) - Fix Steering Adoption stat panel returning string instead of numeric value - Add Playwright E2E test covering full pipeline flow and dashboard verification * fix: add Apache license headers to e2e files, fix gofmt alignment
1 parent b68c102 commit c4ad2cb

13 files changed

Lines changed: 1801 additions & 354 deletions

File tree

‎backend/plugins/q_dev/models/chat_log.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ type QDevChatLog struct {
4444
ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"`
4545
HasSteering bool `json:"hasSteering"`
4646
IsSpecMode bool `json:"isSpecMode"`
47+
CodeReferenceCount int `json:"codeReferenceCount"`
48+
WebLinkCount int `json:"webLinkCount"`
49+
HasFollowupPrompts bool `json:"hasFollowupPrompts"`
4750
}
4851

4952
func (QDevChatLog) TableName() string {

‎backend/plugins/q_dev/models/completion_log.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,16 +26,18 @@ import (
2626
// QDevCompletionLog stores parsed data from GenerateCompletions logging events
2727
type QDevCompletionLog struct {
2828
common.NoPKModel
29-
ConnectionId uint64 `gorm:"primaryKey"`
30-
ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"`
31-
RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"`
32-
UserId string `gorm:"index;type:varchar(255)" json:"userId"`
33-
DisplayName string `gorm:"type:varchar(255)" json:"displayName"`
34-
Timestamp time.Time `gorm:"index" json:"timestamp"`
35-
FileName string `gorm:"type:varchar(512)" json:"fileName"`
36-
FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"`
37-
HasCustomization bool `json:"hasCustomization"`
38-
CompletionsCount int `json:"completionsCount"`
29+
ConnectionId uint64 `gorm:"primaryKey"`
30+
ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"`
31+
RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"`
32+
UserId string `gorm:"index;type:varchar(255)" json:"userId"`
33+
DisplayName string `gorm:"type:varchar(255)" json:"displayName"`
34+
Timestamp time.Time `gorm:"index" json:"timestamp"`
35+
FileName string `gorm:"type:varchar(512)" json:"fileName"`
36+
FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"`
37+
HasCustomization bool `json:"hasCustomization"`
38+
CompletionsCount int `json:"completionsCount"`
39+
LeftContextLength int `json:"leftContextLength"`
40+
RightContextLength int `json:"rightContextLength"`
3941
}
4042

4143
func (QDevCompletionLog) TableName() string {

‎backend/plugins/q_dev/models/migrationscripts/20260319_add_logging_fields.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
package migrationscripts
19+
20+
import (
21+
"github.com/apache/incubator-devlake/core/context"
22+
"github.com/apache/incubator-devlake/core/errors"
23+
"github.com/apache/incubator-devlake/helpers/migrationhelper"
24+
"github.com/apache/incubator-devlake/plugins/q_dev/models/migrationscripts/archived"
25+
)
26+
27+
var _ = (*addLoggingFields)(nil)
28+
29+
type addLoggingFields struct{}
30+
31+
func (*addLoggingFields) Up(basicRes context.BasicRes) errors.Error {
32+
return migrationhelper.AutoMigrateTables(
33+
basicRes,
34+
&archived.QDevChatLog{},
35+
&archived.QDevCompletionLog{},
36+
)
37+
}
38+
39+
func (*addLoggingFields) Version() uint64 {
40+
return 20260319000001
41+
}
42+
43+
func (*addLoggingFields) Name() string {
44+
return "Add code_reference_count, web_link_count, has_followup_prompts to chat_log; left/right_context_length to completion_log"
45+
}

‎backend/plugins/q_dev/models/migrationscripts/archived/chat_log.go‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ type QDevChatLog struct {
4343
ActiveFileExtension string `gorm:"type:varchar(50)" json:"activeFileExtension"`
4444
HasSteering bool `json:"hasSteering"`
4545
IsSpecMode bool `json:"isSpecMode"`
46+
CodeReferenceCount int `json:"codeReferenceCount"`
47+
WebLinkCount int `json:"webLinkCount"`
48+
HasFollowupPrompts bool `json:"hasFollowupPrompts"`
4649
}
4750

4851
func (QDevChatLog) TableName() string {

‎backend/plugins/q_dev/models/migrationscripts/archived/completion_log.go‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -25,16 +25,18 @@ import (
2525

2626
type QDevCompletionLog struct {
2727
archived.NoPKModel
28-
ConnectionId uint64 `gorm:"primaryKey"`
29-
ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"`
30-
RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"`
31-
UserId string `gorm:"index;type:varchar(255)" json:"userId"`
32-
DisplayName string `gorm:"type:varchar(255)" json:"displayName"`
33-
Timestamp time.Time `gorm:"index" json:"timestamp"`
34-
FileName string `gorm:"type:varchar(512)" json:"fileName"`
35-
FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"`
36-
HasCustomization bool `json:"hasCustomization"`
37-
CompletionsCount int `json:"completionsCount"`
28+
ConnectionId uint64 `gorm:"primaryKey"`
29+
ScopeId string `gorm:"primaryKey;type:varchar(255)" json:"scopeId"`
30+
RequestId string `gorm:"primaryKey;type:varchar(255)" json:"requestId"`
31+
UserId string `gorm:"index;type:varchar(255)" json:"userId"`
32+
DisplayName string `gorm:"type:varchar(255)" json:"displayName"`
33+
Timestamp time.Time `gorm:"index" json:"timestamp"`
34+
FileName string `gorm:"type:varchar(512)" json:"fileName"`
35+
FileExtension string `gorm:"type:varchar(50)" json:"fileExtension"`
36+
HasCustomization bool `json:"hasCustomization"`
37+
CompletionsCount int `json:"completionsCount"`
38+
LeftContextLength int `json:"leftContextLength"`
39+
RightContextLength int `json:"rightContextLength"`
3840
}
3941

4042
func (QDevCompletionLog) TableName() string {

‎backend/plugins/q_dev/models/migrationscripts/register.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -36,5 +36,6 @@ func All() []plugin.MigrationScript {
3636
new(fixDedupUserTables),
3737
new(resetS3FileMetaProcessed),
3838
new(addLoggingTables),
39+
new(addLoggingFields),
3940
}
4041
}

‎backend/plugins/q_dev/tasks/s3_logging_extractor.go‎

Lines changed: 22 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -280,10 +280,13 @@ type chatLogRequest struct {
280280
type chatLogResponse struct {
281281
RequestID string `json:"requestId"`
282282
AssistantResponse string `json:"assistantResponse"`
283+
FollowupPrompts string `json:"followupPrompts"`
283284
MessageMetadata struct {
284285
ConversationID *string `json:"conversationId"`
285286
UtteranceID *string `json:"utteranceId"`
286287
} `json:"messageMetadata"`
288+
CodeReferenceEvents []json.RawMessage `json:"codeReferenceEvents"`
289+
SupplementaryWebLinksEvent []json.RawMessage `json:"supplementaryWebLinksEvent"`
287290
}
288291

289292
type completionLogRecord struct {
@@ -296,6 +299,8 @@ type completionLogRequest struct {
296299
Timestamp string `json:"timeStamp"`
297300
FileName string `json:"fileName"`
298301
CustomizationArn *string `json:"customizationArn"`
302+
LeftContext string `json:"leftContext"`
303+
RightContext string `json:"rightContext"`
299304
}
300305

301306
type completionLogResponse struct {
@@ -347,6 +352,11 @@ func parseChatRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta, ident
347352
chatLog.UtteranceId = *record.Response.MessageMetadata.UtteranceID
348353
}
349354

355+
// New fields from docs: codeReferenceEvents, supplementaryWebLinksEvent, followupPrompts
356+
chatLog.CodeReferenceCount = len(record.Response.CodeReferenceEvents)
357+
chatLog.WebLinkCount = len(record.Response.SupplementaryWebLinksEvent)
358+
chatLog.HasFollowupPrompts = record.Response.FollowupPrompts != ""
359+
350360
return chatLog, nil
351361
}
352362

@@ -406,16 +416,18 @@ func parseCompletionRecord(raw json.RawMessage, fileMeta *models.QDevS3FileMeta,
406416

407417
userId := normalizeUserId(record.Request.UserID)
408418
return &models.QDevCompletionLog{
409-
ConnectionId: fileMeta.ConnectionId,
410-
ScopeId: fileMeta.ScopeId,
411-
RequestId: record.Response.RequestID,
412-
UserId: userId,
413-
DisplayName: cachedResolveDisplayName(userId, identityClient, cache),
414-
Timestamp: ts,
415-
FileName: record.Request.FileName,
416-
FileExtension: filepath.Ext(record.Request.FileName),
417-
HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "",
418-
CompletionsCount: len(record.Response.Completions),
419+
ConnectionId: fileMeta.ConnectionId,
420+
ScopeId: fileMeta.ScopeId,
421+
RequestId: record.Response.RequestID,
422+
UserId: userId,
423+
DisplayName: cachedResolveDisplayName(userId, identityClient, cache),
424+
Timestamp: ts,
425+
FileName: record.Request.FileName,
426+
FileExtension: filepath.Ext(record.Request.FileName),
427+
HasCustomization: record.Request.CustomizationArn != nil && *record.Request.CustomizationArn != "",
428+
CompletionsCount: len(record.Response.Completions),
429+
LeftContextLength: len(record.Request.LeftContext),
430+
RightContextLength: len(record.Request.RightContext),
419431
}, nil
420432
}
421433

‎e2e/package.json‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
{
2+
"name": "e2e",
3+
"version": "1.0.0",
4+
"description": "",
5+
"main": "index.js",
6+
"scripts": {
7+
"test": "echo \"Error: no test specified\" && exit 1"
8+
},
9+
"keywords": [],
10+
"author": "",
11+
"license": "ISC",
12+
"type": "commonjs",
13+
"dependencies": {
14+
"@playwright/test": "^1.58.2"
15+
}
16+
}

‎e2e/playwright.config.ts‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
*/
17+
18+
import { defineConfig } from '@playwright/test';
19+
20+
export default defineConfig({
21+
testDir: '.',
22+
testMatch: '*.spec.ts',
23+
timeout: 180000,
24+
expect: {
25+
timeout: 10000,
26+
},
27+
use: {
28+
baseURL: 'http://localhost:4000',
29+
screenshot: 'on',
30+
trace: 'on-first-retry',
31+
},
32+
reporter: [['html', { open: 'never' }], ['list']],
33+
projects: [
34+
{
35+
name: 'chromium',
36+
use: { browserName: 'chromium', viewport: { width: 1440, height: 900 } },
37+
},
38+
],
39+
});

0 commit comments

Comments
 (0)