From 8c5a2ca26e6b74d0dc27e1535d3fab685f65cb39 Mon Sep 17 00:00:00 2001 From: Kaska Date: Mon, 18 Nov 2024 18:04:40 -0500 Subject: [PATCH 1/3] removal of demo day crud --- .../knowledge_data/pagerExtended.txt.json | 1 + demo/knowledge_server/pagerExtended.txt | 11 +++++++++++ demo/knowledge_server/test.txt | 14 +------------- demo/presentation-pagerMate-demoDay.txt | 18 ++++++++++++++++++ 4 files changed, 31 insertions(+), 13 deletions(-) create mode 100644 demo/knowledge_server/knowledge_data/pagerExtended.txt.json create mode 100644 demo/knowledge_server/pagerExtended.txt create mode 100644 demo/presentation-pagerMate-demoDay.txt diff --git a/demo/knowledge_server/knowledge_data/pagerExtended.txt.json b/demo/knowledge_server/knowledge_data/pagerExtended.txt.json new file mode 100644 index 0000000..d24ac61 --- /dev/null +++ b/demo/knowledge_server/knowledge_data/pagerExtended.txt.json @@ -0,0 +1 @@ +{"content":"PagerDuty is a service that helps you manage incidents and alerts.\nBe the hero behind the scenes.\nJoin our global team of innovators shaping the future of digital operations with AI. At PagerDuty, you'll tackle complex problems impacting millions, grow your career at the cutting edge of technology, and help build a more equitable world—all in a flexible, inclusive environment. Are you Dutonian?\nThe PagerDuty Operations Cloud, with AI and automation, is key for modern enterprises. It identifies disruptions, mobilizes teams, and streamlines digital operations for competitive success.\nOur values guide us in creating a workplace of belonging where inclusion of different perspectives and understanding of others not only comes first, but drives our success. Everyone, from our executive leadership team to our beloved mascot, Pagey, expresses our values every day in service to our teammates, customers, and community.\nPagerDuty employees are affectionately known as Dutonians. As a Dutonian, you can lead from any role, no matter your title, with support from your peers and a diverse and trusted leadership team. We're building an inclusive workplace that represents the real, everyday people we support around the world.\nWe are thrilled to be recognized as a global technology leader, top-rated workplace, and industry changemaker. PagerDuty is Great Place to Work-certified™, a Fortune Best Workplace for Millennials, a Fortune Best Medium Workplace, a Fortune Best Workplace in Technology, and a top rated product on TrustRadius and G2. \nPagerDuty is committed to creating a diverse environment and is an Equal Employment Opportunity (EEO) employer. PagerDuty provides equal employment opportunities to all qualified applicants without regard to race, color, religion, sex, sexual orientation, gender identity, national origin, age, protected veteran or disabled status, or genetic information.\nPagerDuty is committed to providing reasonable accommodations for qualified individuals with disabilities in our job application process. Should you require accommodation, please email accommodation@pagerduty.com and we will work with you to meet your accessibility needs.\nPagerDuty uses the E-Verify employment verification program.\nPlease note that our Talent Acquisition team only sends emails directly from @pagerduty.com email addresses. We do not request sensitive personal information via email. We do not initiate hiring processes via SMS. Please carefully examine any outreach purporting to represent PagerDuty. Find more helpful information on our blog."} \ No newline at end of file diff --git a/demo/knowledge_server/pagerExtended.txt b/demo/knowledge_server/pagerExtended.txt new file mode 100644 index 0000000..da07f2f --- /dev/null +++ b/demo/knowledge_server/pagerExtended.txt @@ -0,0 +1,11 @@ +PagerDuty is a service that helps you manage incidents and alerts. +Be the hero behind the scenes. +Join our global team of innovators shaping the future of digital operations with AI. At PagerDuty, you'll tackle complex problems impacting millions, grow your career at the cutting edge of technology, and help build a more equitable world—all in a flexible, inclusive environment. Are you Dutonian? +The PagerDuty Operations Cloud, with AI and automation, is key for modern enterprises. It identifies disruptions, mobilizes teams, and streamlines digital operations for competitive success. +Our values guide us in creating a workplace of belonging where inclusion of different perspectives and understanding of others not only comes first, but drives our success. Everyone, from our executive leadership team to our beloved mascot, Pagey, expresses our values every day in service to our teammates, customers, and community. +PagerDuty employees are affectionately known as Dutonians. As a Dutonian, you can lead from any role, no matter your title, with support from your peers and a diverse and trusted leadership team. We're building an inclusive workplace that represents the real, everyday people we support around the world. +We are thrilled to be recognized as a global technology leader, top-rated workplace, and industry changemaker. PagerDuty is Great Place to Work-certified™, a Fortune Best Workplace for Millennials, a Fortune Best Medium Workplace, a Fortune Best Workplace in Technology, and a top rated product on TrustRadius and G2. +PagerDuty is committed to creating a diverse environment and is an Equal Employment Opportunity (EEO) employer. PagerDuty provides equal employment opportunities to all qualified applicants without regard to race, color, religion, sex, sexual orientation, gender identity, national origin, age, protected veteran or disabled status, or genetic information. +PagerDuty is committed to providing reasonable accommodations for qualified individuals with disabilities in our job application process. Should you require accommodation, please email accommodation@pagerduty.com and we will work with you to meet your accessibility needs. +PagerDuty uses the E-Verify employment verification program. +Please note that our Talent Acquisition team only sends emails directly from @pagerduty.com email addresses. We do not request sensitive personal information via email. We do not initiate hiring processes via SMS. Please carefully examine any outreach purporting to represent PagerDuty. Find more helpful information on our blog. \ No newline at end of file diff --git a/demo/knowledge_server/test.txt b/demo/knowledge_server/test.txt index 8e95ce7..f3dfd06 100644 --- a/demo/knowledge_server/test.txt +++ b/demo/knowledge_server/test.txt @@ -1,15 +1,3 @@ This is a test document. It contains multiple sentences. -You can use this to verify that the knowledge library is working correctly. - -PagerDuty is a service that helps you manage incidents and alerts. -Be the hero behind the scenes. -Join our global team of innovators shaping the future of digital operations with AI. At PagerDuty, you'll tackle complex problems impacting millions, grow your career at the cutting edge of technology, and help build a more equitable world—all in a flexible, inclusive environment. Are you Dutonian? -The PagerDuty Operations Cloud, with AI and automation, is key for modern enterprises. It identifies disruptions, mobilizes teams, and streamlines digital operations for competitive success. -Our values guide us in creating a workplace of belonging where inclusion of different perspectives and understanding of others not only comes first, but drives our success. Everyone, from our executive leadership team to our beloved mascot, Pagey, expresses our values every day in service to our teammates, customers, and community. -PagerDuty employees are affectionately known as Dutonians. As a Dutonian, you can lead from any role, no matter your title, with support from your peers and a diverse and trusted leadership team. We're building an inclusive workplace that represents the real, everyday people we support around the world. -We are thrilled to be recognized as a global technology leader, top-rated workplace, and industry changemaker. PagerDuty is Great Place to Work-certified™, a Fortune Best Workplace for Millennials, a Fortune Best Medium Workplace, a Fortune Best Workplace in Technology, and a top rated product on TrustRadius and G2. -PagerDuty is committed to creating a diverse environment and is an Equal Employment Opportunity (EEO) employer. PagerDuty provides equal employment opportunities to all qualified applicants without regard to race, color, religion, sex, sexual orientation, gender identity, national origin, age, protected veteran or disabled status, or genetic information. -PagerDuty is committed to providing reasonable accommodations for qualified individuals with disabilities in our job application process. Should you require accommodation, please email accommodation@pagerduty.com and we will work with you to meet your accessibility needs. -PagerDuty uses the E-Verify employment verification program. -Please note that our Talent Acquisition team only sends emails directly from @pagerduty.com email addresses. We do not request sensitive personal information via email. We do not initiate hiring processes via SMS. Please carefully examine any outreach purporting to represent PagerDuty. Find more helpful information on our blog. \ No newline at end of file +You can use this to verify that the knowledge library is working correctly. \ No newline at end of file diff --git a/demo/presentation-pagerMate-demoDay.txt b/demo/presentation-pagerMate-demoDay.txt new file mode 100644 index 0000000..24ebaa5 --- /dev/null +++ b/demo/presentation-pagerMate-demoDay.txt @@ -0,0 +1,18 @@ +### REPO SHOWCASE +--- +> DemoDay +> Show the knowledge_server with main.go, and knowledge_data both in demo directory +> Show the knowledge directory +> run "go run main.go" from the knowledge_server directory in the demo directory +> localhost:1313 +> upload test file +> show the knowledge_data, json data uploaded update +> search the chat window + +### CONTENT INTAKE +--- +> Content sourced via: https://careers.pagerduty.com/home + +### DOCUMENTATION +--- +> https://pagerduty.atlassian.net/wiki/spaces/~71202008b13a565fe54dbaab73aa9282ffd025/pages/4152229928/PagerMate \ No newline at end of file From bd87e8eb3baf7580bcee1ba8e423564c024a9224 Mon Sep 17 00:00:00 2001 From: Kaska Date: Mon, 18 Nov 2024 18:08:58 -0500 Subject: [PATCH 2/3] removal of demo day crud --- .../demo}/knowledge_server/knowledge_data/pagerExtended.txt.json | 0 {demo => pkg/demo}/knowledge_server/knowledge_data/test.txt.json | 0 {demo => pkg/demo}/knowledge_server/main.go | 0 {demo => pkg/demo}/knowledge_server/pagerExtended.txt | 0 {demo => pkg/demo}/knowledge_server/templates/chat.html | 0 {demo => pkg/demo}/knowledge_server/test.txt | 0 {demo => pkg/demo}/presentation-pagerMate-demoDay.txt | 0 7 files changed, 0 insertions(+), 0 deletions(-) rename {demo => pkg/demo}/knowledge_server/knowledge_data/pagerExtended.txt.json (100%) rename {demo => pkg/demo}/knowledge_server/knowledge_data/test.txt.json (100%) rename {demo => pkg/demo}/knowledge_server/main.go (100%) rename {demo => pkg/demo}/knowledge_server/pagerExtended.txt (100%) rename {demo => pkg/demo}/knowledge_server/templates/chat.html (100%) rename {demo => pkg/demo}/knowledge_server/test.txt (100%) rename {demo => pkg/demo}/presentation-pagerMate-demoDay.txt (100%) diff --git a/demo/knowledge_server/knowledge_data/pagerExtended.txt.json b/pkg/demo/knowledge_server/knowledge_data/pagerExtended.txt.json similarity index 100% rename from demo/knowledge_server/knowledge_data/pagerExtended.txt.json rename to pkg/demo/knowledge_server/knowledge_data/pagerExtended.txt.json diff --git a/demo/knowledge_server/knowledge_data/test.txt.json b/pkg/demo/knowledge_server/knowledge_data/test.txt.json similarity index 100% rename from demo/knowledge_server/knowledge_data/test.txt.json rename to pkg/demo/knowledge_server/knowledge_data/test.txt.json diff --git a/demo/knowledge_server/main.go b/pkg/demo/knowledge_server/main.go similarity index 100% rename from demo/knowledge_server/main.go rename to pkg/demo/knowledge_server/main.go diff --git a/demo/knowledge_server/pagerExtended.txt b/pkg/demo/knowledge_server/pagerExtended.txt similarity index 100% rename from demo/knowledge_server/pagerExtended.txt rename to pkg/demo/knowledge_server/pagerExtended.txt diff --git a/demo/knowledge_server/templates/chat.html b/pkg/demo/knowledge_server/templates/chat.html similarity index 100% rename from demo/knowledge_server/templates/chat.html rename to pkg/demo/knowledge_server/templates/chat.html diff --git a/demo/knowledge_server/test.txt b/pkg/demo/knowledge_server/test.txt similarity index 100% rename from demo/knowledge_server/test.txt rename to pkg/demo/knowledge_server/test.txt diff --git a/demo/presentation-pagerMate-demoDay.txt b/pkg/demo/presentation-pagerMate-demoDay.txt similarity index 100% rename from demo/presentation-pagerMate-demoDay.txt rename to pkg/demo/presentation-pagerMate-demoDay.txt From 62aa420f49aedace939606724fd4483c8fff8255 Mon Sep 17 00:00:00 2001 From: Kaska Date: Mon, 18 Nov 2024 18:11:28 -0500 Subject: [PATCH 3/3] removal of demo day crud --- HelloDutonian.md => dutonian/HelloDutonian.md | 0 dutonian/deploy/.env.example | 5 + dutonian/deploy/Dockerfile | 34 +++++ dutonian/deploy/docker-compose.yml | 12 ++ knowledge/api/handlers.go | 96 -------------- knowledge/library/document.go | 56 -------- knowledge/library/library.go | 72 ----------- knowledge/storage/storage.go | 121 ------------------ 8 files changed, 51 insertions(+), 345 deletions(-) rename HelloDutonian.md => dutonian/HelloDutonian.md (100%) create mode 100644 dutonian/deploy/.env.example create mode 100644 dutonian/deploy/Dockerfile create mode 100644 dutonian/deploy/docker-compose.yml delete mode 100644 knowledge/api/handlers.go delete mode 100644 knowledge/library/document.go delete mode 100644 knowledge/library/library.go delete mode 100644 knowledge/storage/storage.go diff --git a/HelloDutonian.md b/dutonian/HelloDutonian.md similarity index 100% rename from HelloDutonian.md rename to dutonian/HelloDutonian.md diff --git a/dutonian/deploy/.env.example b/dutonian/deploy/.env.example new file mode 100644 index 0000000..da18410 --- /dev/null +++ b/dutonian/deploy/.env.example @@ -0,0 +1,5 @@ +# Server Configuration +PORT=8080 +ENV=development + +# Add other configuration variables as needed \ No newline at end of file diff --git a/dutonian/deploy/Dockerfile b/dutonian/deploy/Dockerfile new file mode 100644 index 0000000..a48ab37 --- /dev/null +++ b/dutonian/deploy/Dockerfile @@ -0,0 +1,34 @@ +# Build stage +FROM golang:1.22-alpine3.19 AS builder + +# Add security updates and CA certificates +RUN apk update && \ + apk add --no-cache ca-certificates tzdata && \ + update-ca-certificates + +# Create non-root user +RUN adduser -D -g '' appuser + +WORKDIR /app + +# Copy go mod files +COPY go.mod ./ +COPY go.sum ./ +RUN go mod download + +# Copy source code +COPY . . + +# Build the binary with security flags +RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags="-w -s" -o server + +# Final stage +FROM scratch + +# Import from builder +COPY --from=builder /usr/share/zoneinfo /usr/share/zoneinfo +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +EXPOSE 8080 + +CMD ["./server"] \ No newline at end of file diff --git a/dutonian/deploy/docker-compose.yml b/dutonian/deploy/docker-compose.yml new file mode 100644 index 0000000..34402ad --- /dev/null +++ b/dutonian/deploy/docker-compose.yml @@ -0,0 +1,12 @@ +version: '3.8' + +services: + app: + build: + context: .. + dockerfile: deploy/Dockerfile + ports: + - "8080:8080" + environment: + - ENV_FILE=.env + restart: unless-stopped \ No newline at end of file diff --git a/knowledge/api/handlers.go b/knowledge/api/handlers.go deleted file mode 100644 index 0ffbf2e..0000000 --- a/knowledge/api/handlers.go +++ /dev/null @@ -1,96 +0,0 @@ -package api - -import ( - "encoding/json" - "net/http" - "strings" - - "github.com/octokas/go-ai/knowledge/library" -) - -type UploadHandler struct { - lib *library.Library -} - -// type ChatHandler struct { -// lib *library.Library -// } - -// PLAIN UPLOADER -// func NewChatHandler(lib *library.Library) *ChatHandler { -// return &ChatHandler{lib: lib} -// } - -// CHAT HANDLER with fancy prompt -// func NewChatHandler(lib *library.Library) *ChatHandler { -// return &ChatHandler{lib: lib} -// } - -func NewChatHandler(lib *library.Library) http.HandlerFunc { - return func(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - var request struct { - Message string `json:"message"` - } - - if err := json.NewDecoder(r.Body).Decode(&request); err != nil { - http.Error(w, "Invalid request", http.StatusBadRequest) - return - } - - // Search the knowledge base - results := lib.Search(request.Message) - - // Prepare response - response := "Based on our knowledge base: " - if len(results) > 0 { - response += strings.Join(results, "\n\n") - } else { - response = "I couldn't find any relevant information in the knowledge base." - } - - w.Header().Set("Content-Type", "application/json") - json.NewEncoder(w).Encode(map[string]string{ - "response": response, - }) - } -} - -func NewUploadHandler(lib *library.Library) *UploadHandler { - return &UploadHandler{lib: lib} -} - -func (h *UploadHandler) HandleUpload(w http.ResponseWriter, r *http.Request) { - if r.Method != http.MethodPost { - http.Error(w, "Method not allowed", http.StatusMethodNotAllowed) - return - } - - file, header, err := r.FormFile("document") - if err != nil { - http.Error(w, "Error reading file", http.StatusBadRequest) - return - } - defer file.Close() - - // Read file content - content := make([]byte, header.Size) - if _, err := file.Read(content); err != nil { - http.Error(w, "Error reading file content", http.StatusInternalServerError) - return - } - - // Add to library - err = h.lib.AddDocument(header.Filename, string(content)) - if err != nil { - http.Error(w, "Error adding document to library", http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusOK) - json.NewEncoder(w).Encode(map[string]string{"status": "success"}) -} diff --git a/knowledge/library/document.go b/knowledge/library/document.go deleted file mode 100644 index 90185b8..0000000 --- a/knowledge/library/document.go +++ /dev/null @@ -1,56 +0,0 @@ -package library - -import ( - "strings" -) - -type Document struct { - content string - words map[string]int - sentences []string -} - -func NewDocument(content string) *Document { - doc := &Document{ - content: content, - words: make(map[string]int), - } - doc.process() - return doc -} - -func (d *Document) process() { - // Split content into sentences (simplified) - d.sentences = strings.Split(d.content, ".") - - // Build word frequency map - words := strings.Fields(strings.ToLower(d.content)) - for _, word := range words { - d.words[word]++ - } -} - -func (d *Document) MatchesQuery(queryTerms []string) bool { - for _, term := range queryTerms { - if _, exists := d.words[term]; !exists { - return false - } - } - return true -} - -func (d *Document) GetRelevantExcerpts(queryTerms []string) []string { - relevant := []string{} - - for _, sentence := range d.sentences { - sentenceLower := strings.ToLower(sentence) - for _, term := range queryTerms { - if strings.Contains(sentenceLower, term) { - relevant = append(relevant, strings.TrimSpace(sentence)) - break - } - } - } - - return relevant -} diff --git a/knowledge/library/library.go b/knowledge/library/library.go deleted file mode 100644 index c25c969..0000000 --- a/knowledge/library/library.go +++ /dev/null @@ -1,72 +0,0 @@ -package library - -import ( - "strings" - "sync" - - "github.com/octokas/go-ai/knowledge/storage" -) - -type Library struct { - documents map[string]*Document - storage storage.StorageProvider - mutex sync.RWMutex -} - -func NewLibrary(storage storage.StorageProvider) *Library { - return &Library{ - documents: make(map[string]*Document), - storage: storage, - } -} - -func (l *Library) AddDocument(id string, content string) error { - doc := NewDocument(content) - - l.mutex.Lock() - defer l.mutex.Unlock() - - if err := l.storage.Save(id, content); err != nil { - return err - } - - l.documents[id] = doc - return nil -} - -func (l *Library) LoadAllDocuments() error { - l.mutex.Lock() - defer l.mutex.Unlock() - - ids, err := l.storage.List() - if err != nil { - return err - } - - for _, id := range ids { - content, err := l.storage.Load(id) - if err != nil { - return err - } - - l.documents[id] = NewDocument(content) - } - - return nil -} - -func (l *Library) Search(query string) []string { - l.mutex.RLock() - defer l.mutex.RUnlock() - - results := []string{} - queryTerms := strings.Fields(strings.ToLower(query)) - - for _, doc := range l.documents { - if doc.MatchesQuery(queryTerms) { - results = append(results, doc.GetRelevantExcerpts(queryTerms)...) - } - } - - return results -} diff --git a/knowledge/storage/storage.go b/knowledge/storage/storage.go deleted file mode 100644 index 143f0ac..0000000 --- a/knowledge/storage/storage.go +++ /dev/null @@ -1,121 +0,0 @@ -package storage - -import ( - "encoding/json" - "errors" - "os" - "path/filepath" - "sync" -) - -// StorageProvider defines the interface for document storage -type StorageProvider interface { - Save(id string, content string) error - Load(id string) (string, error) - Delete(id string) error - List() ([]string, error) -} - -// FileStorage implements StorageProvider using the local filesystem -type FileStorage struct { - basePath string - mutex sync.RWMutex -} - -// NewFileStorage creates a new FileStorage instance -func NewFileStorage(basePath string) (*FileStorage, error) { - // Create base directory if it doesn't exist - if err := os.MkdirAll(basePath, 0755); err != nil { - return nil, err - } - - return &FileStorage{ - basePath: basePath, - }, nil -} - -// Save stores a document to the filesystem -func (fs *FileStorage) Save(id string, content string) error { - fs.mutex.Lock() - defer fs.mutex.Unlock() - - filePath := filepath.Join(fs.basePath, id+".json") - - doc := struct { - Content string `json:"content"` - }{ - Content: content, - } - - data, err := json.Marshal(doc) - if err != nil { - return err - } - - return os.WriteFile(filePath, data, 0644) -} - -// Load retrieves a document from the filesystem -func (fs *FileStorage) Load(id string) (string, error) { - fs.mutex.RLock() - defer fs.mutex.RUnlock() - - filePath := filepath.Join(fs.basePath, id+".json") - - data, err := os.ReadFile(filePath) - if err != nil { - if os.IsNotExist(err) { - return "", errors.New("document not found") - } - return "", err - } - - var doc struct { - Content string `json:"content"` - } - - if err := json.Unmarshal(data, &doc); err != nil { - return "", err - } - - return doc.Content, nil -} - -// Delete removes a document from the filesystem -func (fs *FileStorage) Delete(id string) error { - fs.mutex.Lock() - defer fs.mutex.Unlock() - - filePath := filepath.Join(fs.basePath, id+".json") - - if err := os.Remove(filePath); err != nil { - if os.IsNotExist(err) { - return errors.New("document not found") - } - return err - } - - return nil -} - -// List returns all document IDs in storage -func (fs *FileStorage) List() ([]string, error) { - fs.mutex.RLock() - defer fs.mutex.RUnlock() - - files, err := os.ReadDir(fs.basePath) - if err != nil { - return nil, err - } - - var ids []string - for _, file := range files { - if !file.IsDir() && filepath.Ext(file.Name()) == ".json" { - // Remove the .json extension to get the ID - id := file.Name()[:len(file.Name())-5] - ids = append(ids, id) - } - } - - return ids, nil -}