Skip to content

Commit c0e6e0a

Browse files
author
Sergey Egorov
committed
Initial public commit
0 parents  commit c0e6e0a

File tree

128 files changed

+6885
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

128 files changed

+6885
-0
lines changed

‎.github/workflows/build.yml

Lines changed: 30 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,30 @@
1+
# This workflow will build a golang project
2+
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-go
3+
4+
name: Build
5+
6+
on:
7+
push:
8+
branches:
9+
- 'master'
10+
pull_request:
11+
branches:
12+
- 'master'
13+
14+
jobs:
15+
16+
build:
17+
runs-on: ubuntu-latest
18+
steps:
19+
- uses: actions/checkout@v4
20+
21+
- name: Set up Go
22+
uses: actions/setup-go@v4
23+
with:
24+
go-version: '1.22.8'
25+
26+
- name: Build
27+
run: go build -v ./...
28+
29+
- name: Test
30+
run: go test -v ./...

‎.gitignore

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
.*
2+
!.gitignore
3+
!.keepme
4+
testdata/tmp
5+
/swamp
6+
/swamp-ui-dev
7+
/cmd/swamp/swamp
8+
/cmd/swamp-ui-dev/swamp-ui-dev
9+
vendor/
10+
cover.out
11+
cover.html

‎20231122_100028.jpg

4.26 MB
Loading

‎LICENSE

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,21 @@
1+
MIT License
2+
3+
Copyright (c) 2024 cloudcopper
4+
5+
Permission is hereby granted, free of charge, to any person obtaining a copy
6+
of this software and associated documentation files (the "Software"), to deal
7+
in the Software without restriction, including without limitation the rights
8+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
copies of the Software, and to permit persons to whom the Software is
10+
furnished to do so, subject to the following conditions:
11+
12+
The above copyright notice and this permission notice shall be included in all
13+
copies or substantial portions of the Software.
14+
15+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
SOFTWARE.

‎Makefile

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
.PHONY:all
2+
all: swamp
3+
4+
SRC = $(shell find . -path '*.go' -or -path '*.html')
5+
swamp: ${SRC}
6+
CGO_ENABLED=0 go build -ldflags="-s -w" ./cmd/swamp
7+
CGO_ENABLED=0 go build -ldflags="-s -w" ./cmd/swamp-ui-dev
8+
9+
.PHONY:test
10+
test:
11+
CGO_ENABLED=0 go test ./...
12+
13+
.PHONY:coverage
14+
coverage:
15+
CGO_ENABLED=0 go test -coverprofile cover.out ./...
16+
go tool cover -html cover.out -o cover.html
17+
18+
.PHONY: crit
19+
crit:
20+
gocritic check -enableAll ./...

‎README.md

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
Swamp - Only minimalistic artifacts storage
2+
===========================================
3+
- No external database needed
4+
- Simpliest UI
5+
- Use of arbitrary directory as per project artifacts inputs
6+
- Yaml config with anchors/aliases support (see [examples/six-repos/six-repos.yml](https://github.com/cloudcopper/swamp/blob/master/examples/six-repos/six-repos.yml))
7+
- Artifacts retention
8+
- Artifacts validation
9+
- Artifacts metadata
10+
- Repo metadata
11+
12+
Quick start
13+
-----------
14+
15+
* Install swamp ```go install github.com/cloudcopper/swamp/cmd/swamp@latest``` or download from [Releases](https://github.com/cloudcopper/swamp/releases)
16+
* Create config file ```swamp_repos.yml```:
17+
```
18+
project-name:
19+
name: "Project Name"
20+
description: "Project description text text text"
21+
input: /home/user/tmp/project-name/
22+
storage: /home/user/tmp/releases/project-name/"
23+
retention: 1h
24+
```
25+
* Run swamp ```swamp```
26+
* Open webui at http://localhost:8080
27+
* Create artifact:
28+
```
29+
# Create artifact
30+
# ===============
31+
# have something to be inside artifact
32+
dd if=/dev/urandom bs=1k count=64 of=file1.bin
33+
dd if=/dev/urandom bs=1k count=64 of=file2.bin
34+
dd if=/dev/urandom bs=1k count=64 of=file3.bin
35+
# create optional metadata
36+
export > _export.txt
37+
# create optional _createdAt.txt
38+
date +%s > _createdAt.txt
39+
# create checksum of artifact files
40+
sha256sum file1.bin file2.bin file3.bin _export.txt _createdAt.txt > _checksum.txt
41+
42+
# Deliver artifact
43+
# ================
44+
mkdir /home/user/tmp/project-name/artifact-id
45+
mv file1.bin file2.bin file3.bin _createdAt /home/user/tmp/project-name/artifact-id
46+
mv _checksum.txt /home/user/tmp/project-name/artifact-id/$(sha256sum _checksum.txt|cut -d' ' -f1).sha256sum)
47+
48+
# Now the swamp would detect the new artifact with id 'artifact-id'
49+
# Check its checksum file
50+
# Move artifact to storage
51+
# And update data
52+
```
53+
* Refresh the http://localhost:8080
54+
55+
How swamp detects new artifacts
56+
-------------------------------
57+
It uses [fsnotify](https://github.com/fsnotify/fsnotify) to monitor the project's input directory
58+
for new files. When a new *.sha256sum file is detected, and its name matches the checksum of its
59+
contents, all files listed within are included as part of the artifact. The directory containing
60+
the checksum file serves as the artifact ID.
61+
62+
Example:
63+
* for project input: ```/tftproot/my_project/```
64+
* and checksum file: ```/tftproot/my_project/rel3.0.0a/f843944c4c15009a3cdc39bf3dfc30c6adbc98bf5b3e056d429f04f1b4ad306b.sha256sum```
65+
* the artifact id would be ```rel3.0.0a```
66+
67+
How to customize
68+
----------------
69+
See [CUSTOM](CUSTOM.md)
70+
71+
Current screenshots
72+
-------------------
73+
![](./frontpage.png)
74+
![](./repo.png)
75+
![](./artifact.png)
76+
77+
Design diagram
78+
--------------
79+
![](./20231122_100028.jpg)
80+
81+
TODO
82+
----
83+
See [TODO](TODO.md)
84+

‎TODO.md

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
TODO
2+
====
3+
- CUSTOM.md - how to customize
4+
- Correct swamp-intro and readme
5+
- Release v0.1.0
6+
7+
- broken artifacts shall not be possible to download within single click/direct url
8+
- broken files shall not be possible to download within single click/direct url
9+
-- figure out how to make it complex for automation - i.e. present link with some random value instead of normal artifact id
10+
11+
- layered fs with afero instead of custom?
12+
13+
- nicer ui
14+
-- main page - artifacts pagination?
15+
-- repo page - artifacts pagination? calendar separation?
16+
-- about page ?
17+
18+
- tests - increase test coverage
19+
- better configuration (atm params are at package level and it might not works well with massive testing or proper DI)
20+
21+
- handle manual artifact removal from artifact storage
22+
- handle manual artifact adding to artifact storage
23+
24+
- access log
25+
- input web (the way to put over http new artifacts)
26+
- abstract out storage
27+
-- currently it is filesystem but may it be more flexible? minio?
28+
29+
- gorm -> goent ???
30+
- uber fx or google wire ???
31+
32+
- archetypes for different artifacts/repos???
33+
34+
- meta filter at the page
35+
- meta search

‎adapters/basic_artifact_storage_adapter.go

Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
package adapters
2+
3+
import (
4+
"fmt"
5+
"log/slog"
6+
"os"
7+
"path/filepath"
8+
"strconv"
9+
"strings"
10+
"time"
11+
12+
"github.com/cloudcopper/swamp/domain/errors"
13+
"github.com/cloudcopper/swamp/domain/models"
14+
"github.com/cloudcopper/swamp/lib"
15+
"github.com/cloudcopper/swamp/ports"
16+
"github.com/spf13/afero"
17+
)
18+
19+
type BasicArtifactStorageAdapter struct {
20+
log ports.Logger
21+
fs ports.FS
22+
}
23+
24+
func NewBasicArtifactStorageAdapter(log ports.Logger, f ports.FS) (*BasicArtifactStorageAdapter, error) {
25+
log = log.With(slog.String("entity", "BasicArtifactStorageAdapter"))
26+
s := &BasicArtifactStorageAdapter{
27+
log: log,
28+
fs: f,
29+
}
30+
31+
return s, nil
32+
}
33+
34+
func (s *BasicArtifactStorageAdapter) NewArtifact(src ports.FS, input string, artifacts []string, storage string, id models.ArtifactID) (*ports.NewArtifactInfo, error) {
35+
lib.Assert(storage != "")
36+
lib.Assert(id != "")
37+
lib.Assert(len(artifacts) >= 1)
38+
log, dst := s.log, s.fs
39+
log = log.With(slog.Any("storage", storage), slog.String("artifactID", string(id)))
40+
log.Info("add artifacts", slog.Any("input", input), slog.Any("files", artifacts))
41+
42+
exist, _ := afero.DirExists(dst, storage)
43+
if !exist {
44+
return nil, lib.ErrNoSuchDirectory{Path: storage}
45+
}
46+
47+
dest := filepath.Join(storage, string(id))
48+
exist, _ = afero.DirExists(dst, dest)
49+
if exist {
50+
return nil, errors.ErrArtifactAlreadyExists{Path: dest}
51+
}
52+
if err := dst.MkdirAll(dest, os.ModePerm); err != nil {
53+
return nil, err
54+
}
55+
56+
// Move all artifacts
57+
size := int64(0)
58+
for _, fileName := range artifacts {
59+
// The input must be sanitized already!!!
60+
lib.Assert(lib.IsSecureFileName(fileName))
61+
lib.Assert(strings.HasPrefix(fileName, input))
62+
63+
// Using input, fileName and id to detect path withing artifact
64+
name := fileName
65+
name = strings.TrimPrefix(name, input)
66+
name = strings.TrimPrefix(name, string(os.PathSeparator))
67+
name = strings.TrimPrefix(name, id+string(os.PathSeparator))
68+
dir, file := filepath.Split(name)
69+
dest := filepath.Join(dest, dir)
70+
if dir != "" {
71+
if err := dst.MkdirAll(dest, os.ModePerm); err != nil {
72+
return nil, err
73+
}
74+
}
75+
newpath := filepath.Join(dest, file)
76+
// Move single artifact
77+
if err := lib.MoveFile(src, fileName, dst, newpath); err != nil {
78+
return nil, err
79+
}
80+
size += lib.FileSize(dst, newpath)
81+
}
82+
83+
// Optional create file _createdAt.txt containing epoch time.
84+
// It can be part of artifacts as well.
85+
// In such case the creation time would be preserved by checksum file.
86+
// Can be created by ```date +%s > _createdAt.txt```
87+
now := time.Now().UTC().Unix()
88+
file := filepath.Join(dest, "_createdAt.txt")
89+
if err := lib.CreateFile(dst, file, fmt.Sprintf("%v", now)); lib.NoSuchFile(dst, file) && err != nil {
90+
log.Warn("unable to create", slog.String("file", file), slog.Any("err", err))
91+
}
92+
93+
// Read back creation time
94+
a, err := afero.ReadFile(dst, file)
95+
if err != nil {
96+
log.Warn("unable to read", slog.String("file", file), slog.Any("err", err))
97+
}
98+
// Once external creation time might be created with tailing \n or even more
99+
// parse only leading digits and ignore rest
100+
t, err := strconv.ParseInt(lib.LeadingDigits(string(a)), 10, 64)
101+
if err != nil {
102+
log.Warn("unable convert creation time", slog.Any("err", err))
103+
}
104+
createdAt := t
105+
106+
info := &ports.NewArtifactInfo{
107+
Size: size,
108+
CreatedAt: createdAt,
109+
}
110+
111+
return info, nil
112+
}
113+
114+
func (s *BasicArtifactStorageAdapter) RemoveArtifact(storage string, artifactID models.ArtifactID) error {
115+
path := filepath.Join(storage, artifactID)
116+
err := s.fs.RemoveAll(path)
117+
return err
118+
}
119+
120+
func (s *BasicArtifactStorageAdapter) OpenFile(storage string, artifactID models.ArtifactID, filename string) (ports.File, error) {
121+
path := filepath.Join(storage, artifactID, filename)
122+
f, err := s.fs.OpenFile(path, os.O_RDONLY, 0)
123+
return f, err
124+
}
125+
126+
func (s *BasicArtifactStorageAdapter) Close() {
127+
log := s.log
128+
log.Info("closing")
129+
}

0 commit comments

Comments
 (0)