From d1970cddfc78eb70e263a58185b994cb4936241a Mon Sep 17 00:00:00 2001 From: Muhamad Azmy Date: Thu, 18 Apr 2019 15:53:22 +0200 Subject: [PATCH] Cherrypick a fix for memory consumption Also rename imports --- apps/core0/bootstrap/bootstrap.go | 12 +- apps/core0/bootstrap/network/dhcp.go | 2 +- apps/core0/bootstrap/network/manager.go | 2 +- apps/core0/bootstrap/nft.go | 2 +- apps/core0/builtin/bridge.go | 6 +- apps/core0/builtin/btrfs/btrfs.go | 2 +- apps/core0/builtin/config.go | 4 +- apps/core0/builtin/disk.go | 4 +- apps/core0/builtin/dmi.go | 2 +- apps/core0/builtin/logger.go | 2 +- apps/core0/builtin/monitor.go | 4 +- apps/core0/builtin/nft.go | 4 +- apps/core0/builtin/power.go | 2 +- apps/core0/builtin/pprof.go | 2 +- apps/core0/builtin/rtinfo.go | 2 +- apps/core0/builtin/web.go | 2 +- apps/core0/helper/filesystem/filesystem.go | 6 +- apps/core0/helper/filesystem/restic.go | 2 +- apps/core0/helper/socat/socat.go | 2 +- apps/core0/local.go | 6 +- apps/core0/logger/config.go | 8 +- apps/core0/logger/ledis.go | 6 +- apps/core0/logger/logger.go | 2 +- apps/core0/logger/streamer.go | 4 +- apps/core0/main.go | 32 +- apps/core0/options/options.go | 2 +- apps/core0/output.go | 4 +- apps/core0/screen/screen.go | 2 +- apps/core0/stats/ledis.go | 4 +- apps/core0/stats/stats.go | 2 +- apps/core0/subsys/cgroups/api.go | 2 +- apps/core0/subsys/cgroups/cgroups.go | 2 +- apps/core0/subsys/containers/backup.go | 4 +- apps/core0/subsys/containers/comm.go | 8 +- apps/core0/subsys/containers/container.go | 8 +- apps/core0/subsys/containers/filesystem.go | 8 +- apps/core0/subsys/containers/manager.go | 14 +- apps/core0/subsys/containers/network.go | 4 +- apps/core0/subsys/containers/zerotier.go | 2 +- apps/core0/subsys/kvm/events.go | 6 +- apps/core0/subsys/kvm/filesystem.go | 8 +- apps/core0/subsys/kvm/manager.go | 10 +- apps/core0/subsys/kvm/manager_arm.go | 4 +- apps/core0/subsys/kvm/network.go | 2 +- apps/core0/transport/channel.go | 2 +- apps/core0/transport/sink.go | 2 +- apps/coreX/backend.go | 4 +- apps/coreX/bootstrap/bootstrap.go | 8 +- apps/coreX/bootstrap/plugin.go | 4 +- apps/coreX/main.go | 12 +- apps/corectl/container.go | 2 +- apps/corectl/execute.go | 2 +- apps/corectl/info.go | 2 +- apps/corectl/job.go | 2 +- apps/corectl/ping.go | 2 +- apps/corectl/power.go | 2 +- apps/corectl/response.go | 2 +- apps/corectl/statistics.go | 2 +- apps/corectl/transport.go | 2 +- apps/redis-proxy/main.go | 2 +- apps/redis-proxy/proxy.go | 2 +- base/builtin/bash.go | 2 +- base/builtin/fs.go | 2 +- base/builtin/get_aggregated_stats.go | 2 +- base/builtin/info.go | 4 +- base/builtin/ip.go | 2 +- base/builtin/job.go | 2 +- base/builtin/ping.go | 4 +- base/builtin/process.go | 2 +- base/builtin/subscribe.go | 4 +- base/nft/apply.go | 2 +- base/nft/parser.go | 2 +- base/pm/builtin_test.go | 2 +- base/pm/builtinprocess.go | 2 +- base/pm/builtinprocess_test.go | 2 +- base/pm/containerprocess.go | 2 +- base/pm/extensionprocess.go | 4 +- base/pm/handlers.go | 2 +- base/pm/job.go | 2 +- base/pm/job_test.go | 2 +- base/pm/pm.go | 6 +- base/pm/process.go | 2 +- base/pm/runnerhook.go | 2 +- base/pm/stream/buffer.go | 3 - base/pm/stream/buffer_test.go | 2 +- base/pm/stream/consumer_test.go | 405 +++++++++++++++++- base/pm/stream/input.go | 198 ++++++--- base/pm/stream/meta_test.go | 3 +- base/pm/systemprocess.go | 12 +- base/pm/systemprocess_test.go | 2 +- base/settings/include.go | 2 +- base/settings/settings.go | 2 +- base/settings/startup.go | 2 +- .../py-client/zeroos/core0/client/client.py | 2 +- 94 files changed, 727 insertions(+), 234 deletions(-) diff --git a/apps/core0/bootstrap/bootstrap.go b/apps/core0/bootstrap/bootstrap.go index 45282e64..1eae6a68 100644 --- a/apps/core0/bootstrap/bootstrap.go +++ b/apps/core0/bootstrap/bootstrap.go @@ -9,12 +9,12 @@ import ( "github.com/op/go-logging" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/apps/core0/bootstrap/network" - "github.com/zero-os/0-core/apps/core0/options" - "github.com/zero-os/0-core/apps/core0/screen" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/settings" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/apps/core0/bootstrap/network" + "github.com/threefoldtech/0-core/apps/core0/options" + "github.com/threefoldtech/0-core/apps/core0/screen" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/settings" + "github.com/threefoldtech/0-core/base/utils" ) const ( diff --git a/apps/core0/bootstrap/network/dhcp.go b/apps/core0/bootstrap/network/dhcp.go index 89167cbe..903e1649 100644 --- a/apps/core0/bootstrap/network/dhcp.go +++ b/apps/core0/bootstrap/network/dhcp.go @@ -6,7 +6,7 @@ import ( "io/ioutil" "os" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/core0/bootstrap/network/manager.go b/apps/core0/bootstrap/network/manager.go index f8862de0..b6f800fd 100644 --- a/apps/core0/bootstrap/network/manager.go +++ b/apps/core0/bootstrap/network/manager.go @@ -4,7 +4,7 @@ import ( "fmt" "github.com/op/go-logging" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/utils" ) var ( diff --git a/apps/core0/bootstrap/nft.go b/apps/core0/bootstrap/nft.go index 13ee2059..ceed659f 100644 --- a/apps/core0/bootstrap/nft.go +++ b/apps/core0/bootstrap/nft.go @@ -3,7 +3,7 @@ package bootstrap import ( "io/ioutil" - "github.com/zero-os/0-core/base/nft" + "github.com/threefoldtech/0-core/base/nft" ) var ( diff --git a/apps/core0/builtin/bridge.go b/apps/core0/builtin/bridge.go index d71b2a9f..4d8a8380 100644 --- a/apps/core0/builtin/bridge.go +++ b/apps/core0/builtin/bridge.go @@ -12,9 +12,9 @@ import ( "syscall" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/base/nft" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/nft" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/utils" ) type bridgeMgr struct { diff --git a/apps/core0/builtin/btrfs/btrfs.go b/apps/core0/builtin/btrfs/btrfs.go index b7bdd972..26749923 100644 --- a/apps/core0/builtin/btrfs/btrfs.go +++ b/apps/core0/builtin/btrfs/btrfs.go @@ -7,7 +7,7 @@ import ( "strconv" "strings" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) var ( diff --git a/apps/core0/builtin/config.go b/apps/core0/builtin/config.go index d897e25e..e01df2f0 100644 --- a/apps/core0/builtin/config.go +++ b/apps/core0/builtin/config.go @@ -1,8 +1,8 @@ package builtin import ( - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/settings" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/settings" ) type configMgr struct{} diff --git a/apps/core0/builtin/disk.go b/apps/core0/builtin/disk.go index d11fd055..2b59c05f 100644 --- a/apps/core0/builtin/disk.go +++ b/apps/core0/builtin/disk.go @@ -10,8 +10,8 @@ import ( "strings" "syscall" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/utils" "golang.org/x/sys/unix" ) diff --git a/apps/core0/builtin/dmi.go b/apps/core0/builtin/dmi.go index e76fd408..8abc31c5 100644 --- a/apps/core0/builtin/dmi.go +++ b/apps/core0/builtin/dmi.go @@ -8,7 +8,7 @@ import ( "strings" "unicode" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) //DMIType (allowed types 0 -> 42) diff --git a/apps/core0/builtin/logger.go b/apps/core0/builtin/logger.go index a4904934..e68ae38f 100644 --- a/apps/core0/builtin/logger.go +++ b/apps/core0/builtin/logger.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/op/go-logging" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "os" "syscall" ) diff --git a/apps/core0/builtin/monitor.go b/apps/core0/builtin/monitor.go index 376f022d..9789d8fa 100644 --- a/apps/core0/builtin/monitor.go +++ b/apps/core0/builtin/monitor.go @@ -9,8 +9,8 @@ import ( "github.com/shirou/gopsutil/net" ps "github.com/shirou/gopsutil/process" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/utils" "io/ioutil" "path" "strconv" diff --git a/apps/core0/builtin/nft.go b/apps/core0/builtin/nft.go index 0c5cb26f..c25453dc 100644 --- a/apps/core0/builtin/nft.go +++ b/apps/core0/builtin/nft.go @@ -6,8 +6,8 @@ import ( "net" "sync" - "github.com/zero-os/0-core/base/nft" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/nft" + "github.com/threefoldtech/0-core/base/pm" ) type nftMgr struct { diff --git a/apps/core0/builtin/power.go b/apps/core0/builtin/power.go index 5c610988..dbf53a0b 100644 --- a/apps/core0/builtin/power.go +++ b/apps/core0/builtin/power.go @@ -3,7 +3,7 @@ package builtin import ( "syscall" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/core0/builtin/pprof.go b/apps/core0/builtin/pprof.go index da4c925d..9c44a6c9 100644 --- a/apps/core0/builtin/pprof.go +++ b/apps/core0/builtin/pprof.go @@ -2,7 +2,7 @@ package builtin import ( "encoding/json" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "os" "runtime" "runtime/pprof" diff --git a/apps/core0/builtin/rtinfo.go b/apps/core0/builtin/rtinfo.go index 3ef13584..ef07741e 100644 --- a/apps/core0/builtin/rtinfo.go +++ b/apps/core0/builtin/rtinfo.go @@ -6,7 +6,7 @@ import ( "sync" "github.com/pborman/uuid" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const cmdbin = "rtinfo-client" diff --git a/apps/core0/builtin/web.go b/apps/core0/builtin/web.go index 0ff6b2b7..e79d415c 100644 --- a/apps/core0/builtin/web.go +++ b/apps/core0/builtin/web.go @@ -7,7 +7,7 @@ import ( "net/http" "os" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) func init() { diff --git a/apps/core0/helper/filesystem/filesystem.go b/apps/core0/helper/filesystem/filesystem.go index 3a6953fc..87271967 100644 --- a/apps/core0/helper/filesystem/filesystem.go +++ b/apps/core0/helper/filesystem/filesystem.go @@ -14,9 +14,9 @@ import ( "strings" "sync" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" - "github.com/zero-os/0-core/base/settings" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/settings" ) const ( diff --git a/apps/core0/helper/filesystem/restic.go b/apps/core0/helper/filesystem/restic.go index ffaa3ae5..f1319fda 100644 --- a/apps/core0/helper/filesystem/restic.go +++ b/apps/core0/helper/filesystem/restic.go @@ -2,7 +2,7 @@ package filesystem import ( "fmt" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "net/url" ) diff --git a/apps/core0/helper/socat/socat.go b/apps/core0/helper/socat/socat.go index 0ebc49b9..ae5a686e 100644 --- a/apps/core0/helper/socat/socat.go +++ b/apps/core0/helper/socat/socat.go @@ -6,7 +6,7 @@ import ( "strings" "sync" - "github.com/zero-os/0-core/base/nft" + "github.com/threefoldtech/0-core/base/nft" "github.com/op/go-logging" ) diff --git a/apps/core0/local.go b/apps/core0/local.go index d91edf36..33e47454 100644 --- a/apps/core0/local.go +++ b/apps/core0/local.go @@ -3,9 +3,9 @@ package main import ( "encoding/json" "fmt" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/utils" - "github.com/zero-os/0-core/apps/core0/subsys/containers" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/utils" + "github.com/threefoldtech/0-core/apps/core0/subsys/containers" "net" "os" "strconv" diff --git a/apps/core0/logger/config.go b/apps/core0/logger/config.go index 062e24f5..9769c028 100644 --- a/apps/core0/logger/config.go +++ b/apps/core0/logger/config.go @@ -1,10 +1,10 @@ package logger import ( - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" - "github.com/zero-os/0-core/base/settings" - "github.com/zero-os/0-core/apps/core0/transport" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/settings" + "github.com/threefoldtech/0-core/apps/core0/transport" ) var ( diff --git a/apps/core0/logger/ledis.go b/apps/core0/logger/ledis.go index 4bed1420..a2e40b29 100644 --- a/apps/core0/logger/ledis.go +++ b/apps/core0/logger/ledis.go @@ -6,9 +6,9 @@ import ( "sync" "github.com/pborman/uuid" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" - "github.com/zero-os/0-core/apps/core0/transport" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/apps/core0/transport" ) const ( diff --git a/apps/core0/logger/logger.go b/apps/core0/logger/logger.go index 01ae780d..3b4d3fbd 100644 --- a/apps/core0/logger/logger.go +++ b/apps/core0/logger/logger.go @@ -2,7 +2,7 @@ package logger import ( "github.com/op/go-logging" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) var ( diff --git a/apps/core0/logger/streamer.go b/apps/core0/logger/streamer.go index e8e966a2..1941c560 100644 --- a/apps/core0/logger/streamer.go +++ b/apps/core0/logger/streamer.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" - "github.com/zero-os/0-core/base/pm/stream" - "github.com/zero-os/0-core/apps/core0/transport" + "github.com/threefoldtech/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/apps/core0/transport" ) const ( diff --git a/apps/core0/main.go b/apps/core0/main.go index d4ac155e..bfad4caf 100644 --- a/apps/core0/main.go +++ b/apps/core0/main.go @@ -5,26 +5,26 @@ import ( "os" "github.com/op/go-logging" - "github.com/zero-os/0-core/apps/core0/assets" - "github.com/zero-os/0-core/apps/core0/bootstrap" - "github.com/zero-os/0-core/apps/core0/logger" - "github.com/zero-os/0-core/apps/core0/options" - "github.com/zero-os/0-core/apps/core0/screen" - "github.com/zero-os/0-core/apps/core0/stats" - "github.com/zero-os/0-core/apps/core0/subsys/cgroups" - "github.com/zero-os/0-core/apps/core0/subsys/containers" - "github.com/zero-os/0-core/apps/core0/subsys/kvm" - "github.com/zero-os/0-core/base" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/settings" + "github.com/threefoldtech/0-core/apps/core0/assets" + "github.com/threefoldtech/0-core/apps/core0/bootstrap" + "github.com/threefoldtech/0-core/apps/core0/logger" + "github.com/threefoldtech/0-core/apps/core0/options" + "github.com/threefoldtech/0-core/apps/core0/screen" + "github.com/threefoldtech/0-core/apps/core0/stats" + "github.com/threefoldtech/0-core/apps/core0/subsys/cgroups" + "github.com/threefoldtech/0-core/apps/core0/subsys/containers" + "github.com/threefoldtech/0-core/apps/core0/subsys/kvm" + core "github.com/threefoldtech/0-core/base" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/settings" "os/signal" "syscall" - _ "github.com/zero-os/0-core/apps/core0/builtin" - _ "github.com/zero-os/0-core/apps/core0/builtin/btrfs" - "github.com/zero-os/0-core/apps/core0/transport" - _ "github.com/zero-os/0-core/base/builtin" + _ "github.com/threefoldtech/0-core/apps/core0/builtin" + _ "github.com/threefoldtech/0-core/apps/core0/builtin/btrfs" + "github.com/threefoldtech/0-core/apps/core0/transport" + _ "github.com/threefoldtech/0-core/base/builtin" ) var ( diff --git a/apps/core0/options/options.go b/apps/core0/options/options.go index 773508da..f2987fa1 100644 --- a/apps/core0/options/options.go +++ b/apps/core0/options/options.go @@ -3,7 +3,7 @@ package options import ( "flag" "fmt" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/utils" "os" ) diff --git a/apps/core0/output.go b/apps/core0/output.go index f4d3ce0c..a44064e9 100644 --- a/apps/core0/output.go +++ b/apps/core0/output.go @@ -8,8 +8,8 @@ import ( "time" "github.com/op/go-logging" - "github.com/zero-os/0-core/apps/core0/options" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/apps/core0/options" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/core0/screen/screen.go b/apps/core0/screen/screen.go index 4516b5d4..110ad85c 100644 --- a/apps/core0/screen/screen.go +++ b/apps/core0/screen/screen.go @@ -10,7 +10,7 @@ import ( "time" "github.com/op/go-logging" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/core0/stats/ledis.go b/apps/core0/stats/ledis.go index 455c2a5e..d9e71f9d 100644 --- a/apps/core0/stats/ledis.go +++ b/apps/core0/stats/ledis.go @@ -10,8 +10,8 @@ import ( "github.com/op/go-logging" "github.com/patrickmn/go-cache" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/apps/core0/transport" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/apps/core0/transport" ) const ( diff --git a/apps/core0/stats/stats.go b/apps/core0/stats/stats.go index b315676a..914f2a00 100644 --- a/apps/core0/stats/stats.go +++ b/apps/core0/stats/stats.go @@ -2,7 +2,7 @@ package stats import ( "encoding/json" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "math" "time" ) diff --git a/apps/core0/subsys/cgroups/api.go b/apps/core0/subsys/cgroups/api.go index 9111b4a8..8e68d1fc 100644 --- a/apps/core0/subsys/cgroups/api.go +++ b/apps/core0/subsys/cgroups/api.go @@ -3,7 +3,7 @@ package cgroups import ( "encoding/json" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) type GroupArg struct { diff --git a/apps/core0/subsys/cgroups/cgroups.go b/apps/core0/subsys/cgroups/cgroups.go index 1744da54..8f6f08d9 100644 --- a/apps/core0/subsys/cgroups/cgroups.go +++ b/apps/core0/subsys/cgroups/cgroups.go @@ -10,7 +10,7 @@ import ( "syscall" logging "github.com/op/go-logging" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) type mkg func(name string, subsys Subsystem) Group diff --git a/apps/core0/subsys/containers/backup.go b/apps/core0/subsys/containers/backup.go index f70182c1..b65d659a 100644 --- a/apps/core0/subsys/containers/backup.go +++ b/apps/core0/subsys/containers/backup.go @@ -3,8 +3,8 @@ package containers import ( "encoding/json" "fmt" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/apps/core0/helper/filesystem" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/apps/core0/helper/filesystem" "io/ioutil" "net/url" "os" diff --git a/apps/core0/subsys/containers/comm.go b/apps/core0/subsys/containers/comm.go index 675941ae..0061db12 100644 --- a/apps/core0/subsys/containers/comm.go +++ b/apps/core0/subsys/containers/comm.go @@ -5,10 +5,10 @@ import ( "fmt" "io" - "github.com/zero-os/0-core/apps/core0/logger" - "github.com/zero-os/0-core/apps/core0/stats" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/apps/core0/logger" + "github.com/threefoldtech/0-core/apps/core0/stats" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" ) const ( diff --git a/apps/core0/subsys/containers/container.go b/apps/core0/subsys/containers/container.go index f0c7e520..1423dac7 100644 --- a/apps/core0/subsys/containers/container.go +++ b/apps/core0/subsys/containers/container.go @@ -9,10 +9,10 @@ import ( "syscall" "time" - "github.com/zero-os/0-core/apps/core0/logger" - "github.com/zero-os/0-core/apps/core0/subsys/cgroups" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/apps/core0/logger" + "github.com/threefoldtech/0-core/apps/core0/subsys/cgroups" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" ) const ( diff --git a/apps/core0/subsys/containers/filesystem.go b/apps/core0/subsys/containers/filesystem.go index 0fffc63b..21229f9c 100644 --- a/apps/core0/subsys/containers/filesystem.go +++ b/apps/core0/subsys/containers/filesystem.go @@ -13,10 +13,10 @@ import ( "syscall" "github.com/shirou/gopsutil/disk" - "github.com/zero-os/0-core/apps/core0/helper/filesystem" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/settings" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/apps/core0/helper/filesystem" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/settings" + "github.com/threefoldtech/0-core/base/utils" ) const ( diff --git a/apps/core0/subsys/containers/manager.go b/apps/core0/subsys/containers/manager.go index f2723fad..322945b7 100644 --- a/apps/core0/subsys/containers/manager.go +++ b/apps/core0/subsys/containers/manager.go @@ -14,13 +14,13 @@ import ( "github.com/op/go-logging" "github.com/pborman/uuid" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/apps/core0/helper/socat" - "github.com/zero-os/0-core/apps/core0/screen" - "github.com/zero-os/0-core/apps/core0/subsys/cgroups" - "github.com/zero-os/0-core/apps/core0/transport" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/settings" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/apps/core0/helper/socat" + "github.com/threefoldtech/0-core/apps/core0/screen" + "github.com/threefoldtech/0-core/apps/core0/subsys/cgroups" + "github.com/threefoldtech/0-core/apps/core0/transport" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/settings" + "github.com/threefoldtech/0-core/base/utils" ) const ( diff --git a/apps/core0/subsys/containers/network.go b/apps/core0/subsys/containers/network.go index bd3121ef..865186d0 100644 --- a/apps/core0/subsys/containers/network.go +++ b/apps/core0/subsys/containers/network.go @@ -15,8 +15,8 @@ import ( "github.com/pborman/uuid" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/apps/core0/helper/socat" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/apps/core0/helper/socat" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/core0/subsys/containers/zerotier.go b/apps/core0/subsys/containers/zerotier.go index 34efe31a..ee29def3 100644 --- a/apps/core0/subsys/containers/zerotier.go +++ b/apps/core0/subsys/containers/zerotier.go @@ -3,7 +3,7 @@ package containers import ( "encoding/json" "fmt" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "io/ioutil" "path" "strings" diff --git a/apps/core0/subsys/kvm/events.go b/apps/core0/subsys/kvm/events.go index 69636f31..e0784fee 100644 --- a/apps/core0/subsys/kvm/events.go +++ b/apps/core0/subsys/kvm/events.go @@ -9,9 +9,9 @@ import ( "github.com/google/shlex" "github.com/libvirt/libvirt-go" - "github.com/zero-os/0-core/apps/core0/helper/socat" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/apps/core0/helper/socat" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" ) func (m *kvmManager) deviceRemovedFailedHandler(c *libvirt.Connect, d *libvirt.Domain, event *libvirt.DomainEventDeviceRemovalFailed) { diff --git a/apps/core0/subsys/kvm/filesystem.go b/apps/core0/subsys/kvm/filesystem.go index d5c4405d..8f4a155c 100644 --- a/apps/core0/subsys/kvm/filesystem.go +++ b/apps/core0/subsys/kvm/filesystem.go @@ -7,10 +7,10 @@ import ( "path" "syscall" - "github.com/zero-os/0-core/apps/core0/helper/filesystem" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/settings" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/apps/core0/helper/filesystem" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/settings" + "github.com/threefoldtech/0-core/base/utils" yaml "gopkg.in/yaml.v2" ) diff --git a/apps/core0/subsys/kvm/manager.go b/apps/core0/subsys/kvm/manager.go index 93b8752d..63fdf49f 100644 --- a/apps/core0/subsys/kvm/manager.go +++ b/apps/core0/subsys/kvm/manager.go @@ -17,11 +17,11 @@ import ( "github.com/libvirt/libvirt-go" "github.com/op/go-logging" "github.com/pborman/uuid" - "github.com/zero-os/0-core/apps/core0/helper/socat" - "github.com/zero-os/0-core/apps/core0/screen" - "github.com/zero-os/0-core/apps/core0/subsys/containers" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/apps/core0/helper/socat" + "github.com/threefoldtech/0-core/apps/core0/screen" + "github.com/threefoldtech/0-core/apps/core0/subsys/containers" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/utils" ) const ( diff --git a/apps/core0/subsys/kvm/manager_arm.go b/apps/core0/subsys/kvm/manager_arm.go index 3f164ac7..3327e229 100644 --- a/apps/core0/subsys/kvm/manager_arm.go +++ b/apps/core0/subsys/kvm/manager_arm.go @@ -5,8 +5,8 @@ package kvm import ( "github.com/op/go-logging" - "github.com/zero-os/0-core/apps/core0/screen" - "github.com/zero-os/0-core/apps/core0/subsys/containers" + "github.com/threefoldtech/0-core/apps/core0/screen" + "github.com/threefoldtech/0-core/apps/core0/subsys/containers" ) var ( diff --git a/apps/core0/subsys/kvm/network.go b/apps/core0/subsys/kvm/network.go index 3656e5d4..1b29019a 100644 --- a/apps/core0/subsys/kvm/network.go +++ b/apps/core0/subsys/kvm/network.go @@ -12,7 +12,7 @@ import ( "github.com/libvirt/libvirt-go" "github.com/pborman/uuid" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/core0/transport/channel.go b/apps/core0/transport/channel.go index 263b3160..ce6d27cf 100644 --- a/apps/core0/transport/channel.go +++ b/apps/core0/transport/channel.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/garyburd/redigo/redis" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/core0/transport/sink.go b/apps/core0/transport/sink.go index a219a317..a1eb1f0a 100644 --- a/apps/core0/transport/sink.go +++ b/apps/core0/transport/sink.go @@ -5,7 +5,7 @@ import ( "time" "github.com/garyburd/redigo/redis" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/apps/coreX/backend.go b/apps/coreX/backend.go index 9f8a7886..6b07c0ec 100644 --- a/apps/coreX/backend.go +++ b/apps/coreX/backend.go @@ -2,8 +2,8 @@ package main import ( "encoding/json" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" "os" "sync" ) diff --git a/apps/coreX/bootstrap/bootstrap.go b/apps/coreX/bootstrap/bootstrap.go index 92168abe..3b37aa00 100644 --- a/apps/coreX/bootstrap/bootstrap.go +++ b/apps/coreX/bootstrap/bootstrap.go @@ -8,10 +8,10 @@ import ( "github.com/op/go-logging" "github.com/shirou/gopsutil/process" - "github.com/zero-os/0-core/apps/coreX/options" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/settings" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/apps/coreX/options" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/settings" + "github.com/threefoldtech/0-core/base/utils" ) var ( diff --git a/apps/coreX/bootstrap/plugin.go b/apps/coreX/bootstrap/plugin.go index 89607929..d613ce4d 100644 --- a/apps/coreX/bootstrap/plugin.go +++ b/apps/coreX/bootstrap/plugin.go @@ -2,8 +2,8 @@ package bootstrap import ( "fmt" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/utils" "strings" ) diff --git a/apps/coreX/main.go b/apps/coreX/main.go index 61d674eb..548c0357 100644 --- a/apps/coreX/main.go +++ b/apps/coreX/main.go @@ -5,18 +5,18 @@ import ( "os" "github.com/op/go-logging" - "github.com/zero-os/0-core/apps/coreX/bootstrap" - "github.com/zero-os/0-core/apps/coreX/options" - "github.com/zero-os/0-core/base" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/apps/coreX/bootstrap" + "github.com/threefoldtech/0-core/apps/coreX/options" + "github.com/threefoldtech/0-core/base" + "github.com/threefoldtech/0-core/base/pm" "os/signal" "syscall" "encoding/json" - _ "github.com/zero-os/0-core/apps/coreX/builtin" - _ "github.com/zero-os/0-core/base/builtin" + _ "github.com/threefoldtech/0-core/apps/coreX/builtin" + _ "github.com/threefoldtech/0-core/base/builtin" ) const ( diff --git a/apps/corectl/container.go b/apps/corectl/container.go index acc943ed..a192ab44 100644 --- a/apps/corectl/container.go +++ b/apps/corectl/container.go @@ -5,7 +5,7 @@ import ( "fmt" "github.com/codegangsta/cli" "github.com/olekukonko/tablewriter" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "gopkg.in/yaml.v2" "os" "sort" diff --git a/apps/corectl/execute.go b/apps/corectl/execute.go index 73645797..32a6f0cb 100644 --- a/apps/corectl/execute.go +++ b/apps/corectl/execute.go @@ -2,7 +2,7 @@ package main import ( "github.com/codegangsta/cli" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) func system(t Transport, c *cli.Context) { diff --git a/apps/corectl/info.go b/apps/corectl/info.go index 9022c45e..4d07035a 100644 --- a/apps/corectl/info.go +++ b/apps/corectl/info.go @@ -2,7 +2,7 @@ package main import ( "github.com/codegangsta/cli" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) func info(t Transport, cmd string, body ...interface{}) { diff --git a/apps/corectl/job.go b/apps/corectl/job.go index 8ae40d7d..d3c0d9f0 100644 --- a/apps/corectl/job.go +++ b/apps/corectl/job.go @@ -2,7 +2,7 @@ package main import ( "github.com/codegangsta/cli" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) func jobs(t Transport, c *cli.Context) { diff --git a/apps/corectl/ping.go b/apps/corectl/ping.go index 87596e7c..413098a1 100644 --- a/apps/corectl/ping.go +++ b/apps/corectl/ping.go @@ -2,7 +2,7 @@ package main import ( "github.com/codegangsta/cli" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) func ping(t Transport, c *cli.Context) { diff --git a/apps/corectl/power.go b/apps/corectl/power.go index 709a5969..287df8ea 100644 --- a/apps/corectl/power.go +++ b/apps/corectl/power.go @@ -2,7 +2,7 @@ package main import ( "github.com/codegangsta/cli" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) func reboot(t Transport, c *cli.Context) { diff --git a/apps/corectl/response.go b/apps/corectl/response.go index 53c0106e..e68eaf66 100644 --- a/apps/corectl/response.go +++ b/apps/corectl/response.go @@ -3,7 +3,7 @@ package main import ( "encoding/json" "fmt" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "gopkg.in/yaml.v2" "io" "os" diff --git a/apps/corectl/statistics.go b/apps/corectl/statistics.go index 8893c5ae..17702f3e 100644 --- a/apps/corectl/statistics.go +++ b/apps/corectl/statistics.go @@ -4,7 +4,7 @@ import ( "strings" "github.com/codegangsta/cli" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) func statistics(t Transport, c *cli.Context) { diff --git a/apps/corectl/transport.go b/apps/corectl/transport.go index e2873925..e9597050 100644 --- a/apps/corectl/transport.go +++ b/apps/corectl/transport.go @@ -4,7 +4,7 @@ import ( "encoding/json" "github.com/codegangsta/cli" "github.com/pborman/uuid" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "net" "sync" ) diff --git a/apps/redis-proxy/main.go b/apps/redis-proxy/main.go index da8d0a54..473137bd 100644 --- a/apps/redis-proxy/main.go +++ b/apps/redis-proxy/main.go @@ -5,7 +5,7 @@ import ( "github.com/codegangsta/cli" logging "github.com/op/go-logging" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/utils" ) var ( diff --git a/apps/redis-proxy/proxy.go b/apps/redis-proxy/proxy.go index e340eb68..e3a557d9 100644 --- a/apps/redis-proxy/proxy.go +++ b/apps/redis-proxy/proxy.go @@ -7,7 +7,7 @@ import ( "github.com/garyburd/redigo/redis" "github.com/tidwall/redcon" - "github.com/zero-os/0-core/apps/redis-proxy/assets" + "github.com/threefoldtech/0-core/apps/redis-proxy/assets" ) func newPool(socket string) *redis.Pool { diff --git a/base/builtin/bash.go b/base/builtin/bash.go index d5ca2e9c..b037bbf8 100644 --- a/base/builtin/bash.go +++ b/base/builtin/bash.go @@ -1,6 +1,6 @@ package builtin -import "github.com/zero-os/0-core/base/pm" +import "github.com/threefoldtech/0-core/base/pm" func init() { pm.RegisterExtension("bash", "sh", "", []string{"-c", "{script}"}, nil) diff --git a/base/builtin/fs.go b/base/builtin/fs.go index ab35ea88..cc4d4b28 100644 --- a/base/builtin/fs.go +++ b/base/builtin/fs.go @@ -14,7 +14,7 @@ import ( "github.com/patrickmn/go-cache" "github.com/pborman/uuid" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/base/builtin/get_aggregated_stats.go b/base/builtin/get_aggregated_stats.go index 8d04f925..f6390c9b 100644 --- a/base/builtin/get_aggregated_stats.go +++ b/base/builtin/get_aggregated_stats.go @@ -2,7 +2,7 @@ package builtin import ( psutil "github.com/shirou/gopsutil/process" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "os" ) diff --git a/base/builtin/info.go b/base/builtin/info.go index 569b1b06..60c155a6 100644 --- a/base/builtin/info.go +++ b/base/builtin/info.go @@ -7,8 +7,8 @@ import ( "github.com/shirou/gopsutil/host" "github.com/shirou/gopsutil/mem" "github.com/shirou/gopsutil/net" - base "github.com/zero-os/0-core/base" - "github.com/zero-os/0-core/base/pm" + base "github.com/threefoldtech/0-core/base" + "github.com/threefoldtech/0-core/base/pm" "gopkg.in/bufio.v1" "io/ioutil" gonet "net" diff --git a/base/builtin/ip.go b/base/builtin/ip.go index 11bc1d7b..29708983 100644 --- a/base/builtin/ip.go +++ b/base/builtin/ip.go @@ -6,7 +6,7 @@ import ( "net" "github.com/vishvananda/netlink" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) type ipmgr struct{} diff --git a/base/builtin/job.go b/base/builtin/job.go index 4b62ce88..1251bdfb 100644 --- a/base/builtin/job.go +++ b/base/builtin/job.go @@ -3,7 +3,7 @@ package builtin import ( "encoding/json" "fmt" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "syscall" ) diff --git a/base/builtin/ping.go b/base/builtin/ping.go index b0f28a1c..888189f1 100644 --- a/base/builtin/ping.go +++ b/base/builtin/ping.go @@ -2,8 +2,8 @@ package builtin import ( "fmt" - base "github.com/zero-os/0-core/base" - "github.com/zero-os/0-core/base/pm" + base "github.com/threefoldtech/0-core/base" + "github.com/threefoldtech/0-core/base/pm" ) const ( diff --git a/base/builtin/process.go b/base/builtin/process.go index ee3600ff..71b820e2 100644 --- a/base/builtin/process.go +++ b/base/builtin/process.go @@ -4,7 +4,7 @@ import ( "encoding/json" "fmt" "github.com/shirou/gopsutil/process" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" "io/ioutil" "strings" "syscall" diff --git a/base/builtin/subscribe.go b/base/builtin/subscribe.go index ef4ea4a3..e9f86af1 100644 --- a/base/builtin/subscribe.go +++ b/base/builtin/subscribe.go @@ -3,8 +3,8 @@ package builtin import ( "encoding/json" "fmt" - "github.com/zero-os/0-core/base/pm" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm/stream" ) func init() { diff --git a/base/nft/apply.go b/base/nft/apply.go index 90e7d6e0..e507cbd8 100644 --- a/base/nft/apply.go +++ b/base/nft/apply.go @@ -6,7 +6,7 @@ import ( "os" logging "github.com/op/go-logging" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) var ( diff --git a/base/nft/parser.go b/base/nft/parser.go index 3c14b009..905dfb87 100644 --- a/base/nft/parser.go +++ b/base/nft/parser.go @@ -7,7 +7,7 @@ import ( "regexp" "strings" - "github.com/zero-os/0-core/base/pm" + "github.com/threefoldtech/0-core/base/pm" ) //Get gets current nft ruleset diff --git a/base/pm/builtin_test.go b/base/pm/builtin_test.go index 1cf2f31d..172c3273 100644 --- a/base/pm/builtin_test.go +++ b/base/pm/builtin_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) func TestGetProcessFactory(t *testing.T) { diff --git a/base/pm/builtinprocess.go b/base/pm/builtinprocess.go index 03648d9b..08a4dbf1 100644 --- a/base/pm/builtinprocess.go +++ b/base/pm/builtinprocess.go @@ -7,7 +7,7 @@ import ( "runtime/debug" "syscall" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) /* diff --git a/base/pm/builtinprocess_test.go b/base/pm/builtinprocess_test.go index feb1a8eb..91d2ec40 100644 --- a/base/pm/builtinprocess_test.go +++ b/base/pm/builtinprocess_test.go @@ -5,7 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) func TestBuiltInProcess(t *testing.T) { diff --git a/base/pm/containerprocess.go b/base/pm/containerprocess.go index 49d3da60..f560b71f 100644 --- a/base/pm/containerprocess.go +++ b/base/pm/containerprocess.go @@ -9,7 +9,7 @@ import ( "syscall" psutils "github.com/shirou/gopsutil/process" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) //ContainerCommandArguments arguments for container command diff --git a/base/pm/extensionprocess.go b/base/pm/extensionprocess.go index d6dcb99c..3933c77e 100644 --- a/base/pm/extensionprocess.go +++ b/base/pm/extensionprocess.go @@ -5,8 +5,8 @@ import ( "fmt" "syscall" - "github.com/zero-os/0-core/base/pm/stream" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/utils" ) type extensionProcess struct { diff --git a/base/pm/handlers.go b/base/pm/handlers.go index b14d6550..b153870f 100644 --- a/base/pm/handlers.go +++ b/base/pm/handlers.go @@ -1,6 +1,6 @@ package pm -import "github.com/zero-os/0-core/base/pm/stream" +import "github.com/threefoldtech/0-core/base/pm/stream" //Handler defines an interface to receiver the process manager events //A handler can be any object that implements one or many handle methods below diff --git a/base/pm/job.go b/base/pm/job.go index 2b29c5e6..3b2b4383 100644 --- a/base/pm/job.go +++ b/base/pm/job.go @@ -11,7 +11,7 @@ import ( "syscall" "time" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) const ( diff --git a/base/pm/job_test.go b/base/pm/job_test.go index 2b5a0e1b..f7120b22 100644 --- a/base/pm/job_test.go +++ b/base/pm/job_test.go @@ -6,7 +6,7 @@ import ( "testing" "time" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" "github.com/stretchr/testify/assert" ) diff --git a/base/pm/pm.go b/base/pm/pm.go index 5c93b798..de2f1175 100644 --- a/base/pm/pm.go +++ b/base/pm/pm.go @@ -13,9 +13,9 @@ import ( "github.com/op/go-logging" "github.com/pborman/uuid" - "github.com/zero-os/0-core/base/pm/stream" - "github.com/zero-os/0-core/base/settings" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/settings" + "github.com/threefoldtech/0-core/base/utils" ) const ( diff --git a/base/pm/process.go b/base/pm/process.go index 4d723cf9..19e7a0ef 100644 --- a/base/pm/process.go +++ b/base/pm/process.go @@ -3,7 +3,7 @@ package pm import ( "syscall" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) const ( diff --git a/base/pm/runnerhook.go b/base/pm/runnerhook.go index f140fe84..51d27029 100644 --- a/base/pm/runnerhook.go +++ b/base/pm/runnerhook.go @@ -6,7 +6,7 @@ import ( "sync" "time" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) //RunnerHook is a per process event handler diff --git a/base/pm/stream/buffer.go b/base/pm/stream/buffer.go index 6dfde1ed..bdb8d8dd 100644 --- a/base/pm/stream/buffer.go +++ b/base/pm/stream/buffer.go @@ -21,9 +21,6 @@ func NewBuffer(size int) *Buffer { func (b *Buffer) String() string { var strbuf bytes.Buffer for l := b.Front(); l != nil; l = l.Next() { - if strbuf.Len() > 0 { - strbuf.WriteString("\n") - } switch v := l.Value.(type) { case string: strbuf.WriteString(v) diff --git a/base/pm/stream/buffer_test.go b/base/pm/stream/buffer_test.go index 282b1d38..ae44df55 100644 --- a/base/pm/stream/buffer_test.go +++ b/base/pm/stream/buffer_test.go @@ -55,7 +55,7 @@ func TestBufferString(t *testing.T) { expected := "" for i := 10; i < 20; i++ { - expected += fmt.Sprintf("line %d\n", i) + expected += fmt.Sprintf("line %d", i) } expected = strings.TrimSpace(expected) diff --git a/base/pm/stream/consumer_test.go b/base/pm/stream/consumer_test.go index e42f8d4d..077de3f5 100644 --- a/base/pm/stream/consumer_test.go +++ b/base/pm/stream/consumer_test.go @@ -32,6 +32,344 @@ func (_ *testReader) Close() error { return nil } +func TestConsumer_newLineOrEOF(t *testing.T) { + c := consumerImpl{} + s := "hello world" + x := c.newLineOrEOF([]byte(s)) + if ok := assert.Equal(t, 11, x); !ok { + t.Fail() + } + + s = "hello\nworld" + x = c.newLineOrEOF([]byte(s)) + if ok := assert.Equal(t, 5, x); !ok { + t.Fail() + } + + b := []byte("hello\ngood\nworld") + var r []string + for i := 0; i < len(b); i++ { + n := c.newLineOrEOF(b[i:]) + r = append(r, string(b[i:i+n])) + i += n + } + + if ok := assert.Equal(t, []string{"hello", "good", "world"}, r); !ok { + t.Fail() + } +} + +func TestConsumer_processNormalText(t *testing.T) { + var message *Message + h := func(m *Message) { + message = m + } + + c := consumerImpl{ + level: 1, + handler: h, + } + + c.process([]byte("hello world")) + + if ok := assert.NotNil(t, message); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "hello world", message.Message); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, uint16(1), message.Meta.Level()); !ok { + t.Fatal() + } + + message = nil + txt := "hello world\nthis is output of some program\nthat spans many lines\n" + c.process([]byte(txt)) + + if ok := assert.NotNil(t, message); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, txt, message.Message); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, uint16(1), message.Meta.Level()); !ok { + t.Fatal() + } +} + +func TestConsumer_processSingleLineMessage(t *testing.T) { + var messages []*Message + h := func(m *Message) { + messages = append(messages, m) + } + + c := consumerImpl{ + level: 1, + handler: h, + } + + c.process([]byte(`hello world +the folowing line is a single line message +2::this is a single line message`)) + + if ok := assert.Len(t, messages, 2); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "hello world\nthe folowing line is a single line message\n", messages[0].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[0].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "this is a single line message", messages[1].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(2), messages[1].Meta.Level()); !ok { + t.Error() + } + + messages = nil + + c.process([]byte(`hello world +the folowing line is a single line message +2::this is a single line message +followed by some more text +that spans multiple lines`)) + + if ok := assert.Len(t, messages, 3); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "hello world\nthe folowing line is a single line message\n", messages[0].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[0].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "this is a single line message\n", messages[1].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(2), messages[1].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "followed by some more text\nthat spans multiple lines", messages[2].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[2].Meta.Level()); !ok { + t.Error() + } + + messages = nil + + c.process([]byte(`2::this is a single line message +3::followed by some more messages +4::that has some message`)) + + if ok := assert.Len(t, messages, 3); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "this is a single line message\n", messages[0].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(2), messages[0].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "followed by some more messages\n", messages[1].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(3), messages[1].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "that has some message", messages[2].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(4), messages[2].Meta.Level()); !ok { + t.Error() + } +} + +func TestConsumer_processSingleLineMessageMultiBlocks(t *testing.T) { + var messages []*Message + h := func(m *Message) { + messages = append(messages, m) + } + + c := consumerImpl{ + level: 1, + handler: h, + } + + c.process([]byte(`hello world +the folowing line is a single line message +2::this is a single line message`)) + + c.process([]byte(`followed by some more text +that spans multiple lines`)) + + if ok := assert.Len(t, messages, 3); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "hello world\nthe folowing line is a single line message\n", messages[0].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[0].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "this is a single line message", messages[1].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(2), messages[1].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "followed by some more text\nthat spans multiple lines", messages[2].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[2].Meta.Level()); !ok { + t.Error() + } +} + +func TestConsumer_processMultiLineMessage(t *testing.T) { + var messages []*Message + h := func(m *Message) { + messages = append(messages, m) + } + + c := consumerImpl{ + level: 1, + handler: h, + } + + c.process([]byte(`3:::multi line message +with full termination +in one block +::: +`)) + + if ok := assert.Len(t, messages, 1); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "multi line message\nwith full termination\nin one block", messages[0].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(3), messages[0].Meta.Level()); !ok { + t.Error() + } + + messages = nil + c.process([]byte(`a multi line block is coming +3:::multi line message +with full termination +in one block +::: +which is surrounded by normal text`)) + + if ok := assert.Len(t, messages, 3); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "a multi line block is coming\n", messages[0].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[0].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "multi line message\nwith full termination\nin one block", messages[1].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(3), messages[1].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "which is surrounded by normal text", messages[2].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[2].Meta.Level()); !ok { + t.Error() + } +} + +func TestConsumer_processMultiLineMessageMultiBlock(t *testing.T) { + var messages []*Message + h := func(m *Message) { + messages = append(messages, m) + } + + c := consumerImpl{ + level: 1, + handler: h, + } + + c.process([]byte(`a multi line block is coming +30::: +multi line message +with full termination`)) + c.process([]byte(` +in two blocks +::: +which is surrounded by normal text`)) + + if ok := assert.Len(t, messages, 3); !ok { + t.Fatal() + } + + if ok := assert.Equal(t, "a multi line block is coming\n", messages[0].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[0].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "\nmulti line message\nwith full termination\nin two blocks", messages[1].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(30), messages[1].Meta.Level()); !ok { + t.Error() + } + + if ok := assert.Equal(t, "which is surrounded by normal text", messages[2].Message); !ok { + t.Error() + } + + if ok := assert.Equal(t, uint16(1), messages[2].Meta.Level()); !ok { + t.Error() + } + +} + func TestConsumer_OneLine(t *testing.T) { var message *Message h := func(m *Message) { @@ -50,7 +388,7 @@ func TestConsumer_OneLine(t *testing.T) { t.Fatal() } - if ok := assert.Equal(t, "hello world", message.Message); !ok { + if ok := assert.Equal(t, "hello world\n", message.Message); !ok { t.Fatal() } @@ -80,7 +418,7 @@ func TestConsumer_TwoLines(t *testing.T) { t.Fatal() } - if ok := assert.Equal(t, "hello world", messages[0].Message); !ok { + if ok := assert.Equal(t, "hello world\n", messages[0].Message); !ok { t.Error() } @@ -88,7 +426,7 @@ func TestConsumer_TwoLines(t *testing.T) { t.Error() } - if ok := assert.Equal(t, "bye bye world", messages[1].Message); !ok { + if ok := assert.Equal(t, "bye bye world\n", messages[1].Message); !ok { t.Error() } @@ -107,8 +445,8 @@ func TestConsumer_MultiLine(t *testing.T) { wg.Add(1) Consume(&wg, &testReader{ chunks: []string{ - "30:::hello\nworld\n", - ":::\n", + "30:::hello\nworld", + "\n:::\n", }, }, 1, h) @@ -156,7 +494,7 @@ that spans` t.Fatal() } - if ok := assert.Equal(t, "Hello world", messages[0].Message); !ok { + if ok := assert.Equal(t, "Hello world\n", messages[0].Message); !ok { t.Error() } @@ -164,7 +502,7 @@ that spans` t.Error() } - if ok := assert.Equal(t, "this is a single line message", messages[1].Message); !ok { + if ok := assert.Equal(t, "this is a single line message\n", messages[1].Message); !ok { t.Error() } @@ -209,3 +547,56 @@ func TestConsumerNoNewLine(t *testing.T) { t.Fatal() } } + +func TestConsumerJsonOutput(t *testing.T) { + s := `{ + "address": "c4f432413e", + "clock": 1536831437282, + "config": { + "physical": null, + "settings": { + "allowTcpFallbackRelay": true, + "portMappingEnabled": true, + "primaryPort": 9993, + "softwareUpdate": "disable", + "softwareUpdateChannel": "release" + } + }, + "online": true, + "planetWorldId": 149604618, + "planetWorldTimestamp": 1532555817048, + "publicIdentity": "c4f432413e:0:b3566436739189bb7b8198d37b2bc51894cf81622b4bb04a091db4dfa2c00f1c9fb21c09c13f0e20c4ce2888879c7a878b615d585399dc0ba72976636398f776", + "tcpFallbackActive": false, + "version": "1.2.10", + "versionBuild": 0, + "versionMajor": 1, + "versionMinor": 2, + "versionRev": 10 +} +` + + var message *Message + h := func(m *Message) { + message = m + } + + var wg sync.WaitGroup + wg.Add(1) + Consume(&wg, &testReader{ + chunks: []string{"20:::\n", s, ":::\n"}, + }, 1, h) + + wg.Wait() + + if ok := assert.NotNil(t, message); !ok { + t.Fatal() + } + + // if ok := assert.Equal(t, "hello world\n", message.Message); !ok { + // t.Fatal() + // } + + if ok := assert.Equal(t, uint16(20), message.Meta.Level()); !ok { + t.Fatal() + } +} diff --git a/base/pm/stream/input.go b/base/pm/stream/input.go index 397f8337..2798a125 100644 --- a/base/pm/stream/input.go +++ b/base/pm/stream/input.go @@ -1,7 +1,8 @@ package stream import ( - "bufio" + "bytes" + "fmt" "io" "regexp" "strconv" @@ -11,18 +12,34 @@ import ( logging "github.com/op/go-logging" ) +const ( + ioBufferSize = 32 * 1024 + //messageBufferSize = 10 * 1024 +) + var ( - log = logging.MustGetLogger("stream") - pmMsgPattern, _ = regexp.Compile("^(\\d+)(:{2,3})(.*)$") + log = logging.MustGetLogger("stream") + headerPattern = regexp.MustCompile(`^(\d+)(:{2,3})`) + + multiLineTerm = "\n:::\n" + // multiLineHead = ":::\n" + + errNotHeader = fmt.Errorf("not header") ) type consumerImpl struct { level uint16 handler MessageHandler + source io.Reader - multi *Message + multi *bytes.Buffer + multiLevel uint16 +} - buffer *bufio.Reader +type header struct { + level uint16 + multiline bool + length int } //Consume consumes a stream to the end, and calls the handler with the parsed stream messages @@ -30,78 +47,161 @@ func Consume(wg *sync.WaitGroup, source io.ReadCloser, level uint16, handler Mes c := &consumerImpl{ level: level, handler: handler, - buffer: bufio.NewReaderSize(source, 32*1024), + source: source, } + //source.Read go func() { if wg != nil { defer wg.Done() } + defer source.Close() + if err := c.consume(); err != nil { log.Errorf("failed to read stream: %s", err) } - - source.Close() }() } -func (c *consumerImpl) consume() error { - for { - line, err := c.buffer.ReadString('\n') - if err == io.EOF { - if len(line) != 0 { - c.processLine(line) - } - return nil - } else if err != nil { - return err - } - - c.processLine(line) +func (c *consumerImpl) getHeaderFromMatch(m [][]byte) *header { + level, _ := strconv.ParseUint(string(m[1]), 10, 16) + h := header{ + level: uint16(level), } + if len(m[2]) == 3 { + h.multiline = true + } + h.length = len(m[0]) + return &h } -func (c *consumerImpl) processLine(line string) { - line = strings.TrimRight(line, "\n") +//newLineOrEOF will return index of the next \n or EOF (end of file or string) +func (c *consumerImpl) newLineOrEOF(b []byte) int { + var i int + var x byte + for i, x = range b { + if x == '\n' { + break + } + } + if i+1 == len(b) { + i += 1 + } + return i +} +//process process the buffer and return th +func (c *consumerImpl) process(buffer []byte) { if c.multi != nil { - if line == ":::" { - //last, flush mult - c.handler(c.multi) + //we are in a middle of a multi line message + if c.multi.Len() > 0 && c.multi.Bytes()[c.multi.Len()-1] == '\n' && bytes.HasPrefix(buffer, []byte(":::\n")) { + c.handler(&Message{ + Meta: NewMeta(c.multiLevel), + Message: c.multi.String(), + }) c.multi = nil + buffer = buffer[4:] + } else if end := strings.Index(string(buffer), multiLineTerm); end != -1 { + //we found the termination string + c.multi.Write(buffer[:end]) + c.handler(&Message{ + Meta: NewMeta(c.multiLevel), + Message: c.multi.String(), + }) + c.multi = nil + buffer = buffer[end+len(multiLineTerm):] + } else { + c.multi.Write(buffer) return } - - c.multi.Message += "\n" + line - return } - matches := pmMsgPattern.FindStringSubmatch(line) + start := 0 + for i := 0; i < len(buffer); i++ { + m := headerPattern.FindSubmatch(buffer[i:]) + if m == nil { + //no header was found at this position + i += c.newLineOrEOF(buffer[i:]) + continue + } - if matches == nil { - //use default level. - c.handler(&Message{ - Meta: NewMeta(c.level), - Message: line, - }) - } else { - l, _ := strconv.ParseUint(matches[1], 10, 16) - level := uint16(l) - message := matches[3] - - if matches[2] == ":::" { - c.multi = &Message{ - Meta: NewMeta(level), - Message: message, + //if we reach here then we can safely flush what we have in buffer as a message + if i > start { + c.handler(&Message{ + Meta: NewMeta(c.level), + Message: string(buffer[start:i]), + }) + } + + h := c.getHeaderFromMatch(m) + if !h.multiline { + //find next new line or end of line + start = i + h.length //start of text after the header + i = start + c.newLineOrEOF(buffer[start:]) //seek to new line or end of text + + var msg string + if i == len(buffer) { + msg = string(buffer[start:i]) + } else { + msg = string(buffer[start : i+1]) //include the new line } - } else { - //single line message c.handler(&Message{ - Meta: NewMeta(level), - Message: message, + Meta: NewMeta(h.level), + Message: msg, }) + + start = i + 1 + + //TODO: what if the same line output is split !! + + //I think it's better if the single line message must end with new line, in that case we + //need to seek only to new line termination, if not found we save the current state and + //wait for the next feedback + continue } + + log.Debugf("starting multiline") + //multiline message + //read in multi until eof or \n:::\n termination + j := i + h.length + if end := strings.Index(string(buffer[j:]), multiLineTerm); end != -1 { + //we found the termination string + c.handler(&Message{ + Meta: NewMeta(h.level), + Message: string(buffer[j : j+end]), + }) + i = j + end + len(multiLineTerm) // 4 is the width of the termination string + start = i + } else { + c.multiLevel = h.level + c.multi = bytes.NewBuffer(nil) + c.multi.Write(buffer[j:]) + start = len(buffer) + break + } + } + + if start < len(buffer) { + c.handler(&Message{ + Meta: NewMeta(c.level), + Message: string(buffer[start:]), + }) } +} + +func (c *consumerImpl) consume() error { + buffer := make([]byte, ioBufferSize) + for { + size, err := c.source.Read(buffer) //fill what left of the buffer + if err != nil && err != io.EOF { + return err + } + c.process(buffer[:size]) + + if err == io.EOF { + return nil + } + } } diff --git a/base/pm/stream/meta_test.go b/base/pm/stream/meta_test.go index 43d18928..8488684d 100644 --- a/base/pm/stream/meta_test.go +++ b/base/pm/stream/meta_test.go @@ -1,8 +1,9 @@ package stream import ( - "github.com/stretchr/testify/assert" "testing" + + "github.com/stretchr/testify/assert" ) func TestNewMeta(t *testing.T) { diff --git a/base/pm/systemprocess.go b/base/pm/systemprocess.go index 608eff99..1aca22ac 100644 --- a/base/pm/systemprocess.go +++ b/base/pm/systemprocess.go @@ -10,7 +10,7 @@ import ( "syscall" psutils "github.com/shirou/gopsutil/process" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) type SystemCommandArguments struct { @@ -183,12 +183,16 @@ func (p *systemProcessImpl) Run() (ch <-chan *stream.Message, err error) { args = append(args, p.args.Args...) _, err = p.table.RegisterPID(func() (int, error) { ps, err = os.StartProcess(name, args, &attrs) + defer func() { + for _, f := range toClose { + f.Close() + } + }() + if err != nil { return 0, err } - for _, f := range toClose { - f.Close() - } + return ps.Pid, nil }) diff --git a/base/pm/systemprocess_test.go b/base/pm/systemprocess_test.go index 8ee598f5..87160d20 100644 --- a/base/pm/systemprocess_test.go +++ b/base/pm/systemprocess_test.go @@ -4,7 +4,7 @@ import ( "testing" "github.com/stretchr/testify/assert" - "github.com/zero-os/0-core/base/pm/stream" + "github.com/threefoldtech/0-core/base/pm/stream" ) func TestSystemProcess_Run(t *testing.T) { diff --git a/base/settings/include.go b/base/settings/include.go index 412a3c85..67c36a8b 100644 --- a/base/settings/include.go +++ b/base/settings/include.go @@ -6,7 +6,7 @@ import ( "path" "sync" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/utils" ) type After string diff --git a/base/settings/settings.go b/base/settings/settings.go index 796aa6bb..4efa836c 100644 --- a/base/settings/settings.go +++ b/base/settings/settings.go @@ -2,7 +2,7 @@ package settings import ( "github.com/op/go-logging" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/utils" ) var ( diff --git a/base/settings/startup.go b/base/settings/startup.go index dd575093..9a6675f9 100644 --- a/base/settings/startup.go +++ b/base/settings/startup.go @@ -3,7 +3,7 @@ package settings import ( "fmt" - "github.com/zero-os/0-core/base/utils" + "github.com/threefoldtech/0-core/base/utils" ) //StartupCmd startup command config diff --git a/client/py-client/zeroos/core0/client/client.py b/client/py-client/zeroos/core0/client/client.py index a1c20e69..10215df5 100644 --- a/client/py-client/zeroos/core0/client/client.py +++ b/client/py-client/zeroos/core0/client/client.py @@ -3017,7 +3017,7 @@ def __init__(self, host, port=6379, password="", db=0, ssl=True, timeout=None, t if hasattr(socket, 'TCP_KEEPIDLE'): socket_keepalive_options[socket.TCP_KEEPIDLE] = 1 self._redis = redis.Redis(host=host, port=port, password=password, db=db, ssl=ssl, - socket_timeout=socket_timeout, + socket_timeout=socket_timeout, ssl_cert_reqs='none', socket_keepalive=True, socket_keepalive_options=socket_keepalive_options) self._container_manager = ContainerManager(self) self._bridge_manager = BridgeManager(self)