This is an automated email from the ASF dual-hosted git repository. hsluoyz pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/casbin-gateway.git
commit 51f80ec0e1b35287651877998c451055a867fce7 Author: Yang Luo <[email protected]> AuthorDate: Sun Sep 6 02:01:44 2026 +0800 fix(update): take the port the version it replaced is still letting go of --- main.go | 37 ++-- util/port.go | 53 ++++++ version/restart.go | 266 ++++++++++++++++++++++++++++ version/update.go | 22 ++- web/src/components/shared/version-panel.tsx | 49 +++-- web/src/locales/en/data.json | 1 + web/src/locales/zh/data.json | 1 + web/src/types.ts | 2 + 8 files changed, 393 insertions(+), 38 deletions(-) diff --git a/main.go b/main.go index f56cc1f..e1e0f41 100644 --- a/main.go +++ b/main.go @@ -71,10 +71,11 @@ func main() { return } - // The executable an earlier update replaced is only removable once nothing - // is running from it, which is now. - version.CleanupBackup() + // An update that restarted into this process left the version it replaced + // on disk, and this is where that is either finished with or put back. version.Configure(daemonLogPath) + version.BeginStartup() + defer version.RollBackOnPanic() // A URL scheme is only Gateway while it waits for one link. A Gateway that // did not get to hand one over left the scheme registered to itself, and @@ -157,26 +158,26 @@ func main() { addr := conf.GetHttpAddr() // A previous Gateway still holding the management port would keep this one - // from starting, so it is stopped first. - err := util.StopOldInstance(port) - if err != nil { - // A port held by something that is not Gateway stays with it; the bind - // below reports the conflict in full, so a failed kill only needs a note. + // from starting, so it is stopped first - and the port it held stays busy + // for a moment after it goes, which a restart into an update runs straight + // into. beego.Run() would report that as a stack trace, so the port is + // claimed here to explain it in one line instead. + if err := util.ClaimPort(addr, port); err != nil { + // Being the version an update just installed and not being able to + // serve is what a rollback is for: the machine keeps a Gateway. A port + // held by another program is not that, and putting the previous version + // back would take the update away without freeing anything. var foreign *util.ForeignPortError - if !errors.As(err, &foreign) { - fmt.Printf("Casbin Gateway: could not free port %d: %v\n", port, err) + if !errors.As(err, &foreign) && version.RollBackFailedStart(err.Error()) { + return } + util.FatalListenError(port, `change "httpport" in conf/app.conf`, err) } - service.PrintStartupSummary() + // Nothing is left to roll back to from here: this version has the port. + version.FinishStartup() - // beego.Run() binds the management port itself and reports a conflict as a - // stack trace, so the port is probed here first to explain it in one line. - // The gap between probe and bind is unavoidable but harmless: losing the - // race just puts us back to beego's own error. - if err := util.CheckPortAvailableOn(addr, port); err != nil { - util.FatalListenError(port, `change "httpport" in conf/app.conf`, err) - } + service.PrintStartupSummary() // The agent endpoint of a sandboxed agent names this host's own address, and // the configuration naming it outlives this process. diff --git a/util/port.go b/util/port.go index f13786f..8e4e824 100644 --- a/util/port.go +++ b/util/port.go @@ -16,6 +16,7 @@ package util import ( "context" + "errors" "fmt" "net" "os" @@ -40,6 +41,14 @@ const ( // to be told not to try; the systemd unit the installer writes says so. The // value is the conventional sysexits.h EX_CONFIG. ExitCodeFatalConfig = 78 + // portClaimTimeout bounds the wait for a busy port. It is long enough for + // the Gateway being replaced to finish going away, and short enough that a + // port held by another program is reported rather than waited out. + portClaimTimeout = 20 * time.Second + // portStopInterval is how often the holder is looked up again while + // waiting, which costs a netstat each time. + portStopInterval = 2 * time.Second + portClaimInterval = 250 * time.Millisecond ) // ListenTcp binds a TCP listener on all interfaces for the given port. Callers @@ -72,6 +81,50 @@ func CheckPortAvailableOn(addr string, port int) error { return listener.Close() } +// ClaimPort makes the port this process's to bind: a previous Gateway still +// holding it is stopped, and a port that is busy for any other reason is waited +// on rather than given up on at the first try. +// +// The wait is what a restart needs. A Gateway that restarts into an update +// starts while the one it replaces is still going, and the socket that one was +// listening on outlives it by a moment - long enough that netstat already +// reports no listener while the bind still fails. +func ClaimPort(addr string, port int) error { + deadline := time.Now().Add(portClaimTimeout) + waiting := false + // The zero time is in the past, so the holder is looked up before anything + // is waited on. + var nextStop time.Time + + for { + if time.Now().After(nextStop) { + // A port held by something that is not Gateway belongs to that + // program, and no amount of waiting changes that. + if err := StopOldInstance(port); err != nil { + var foreign *ForeignPortError + if errors.As(err, &foreign) { + return err + } + } + nextStop = time.Now().Add(portStopInterval) + } + + err := CheckPortAvailableOn(addr, port) + if err == nil { + return nil + } + if time.Now().After(deadline) { + return err + } + + if !waiting { + waiting = true + fmt.Printf("Casbin Gateway: port %d is still busy, waiting for it to be released\n", port) + } + time.Sleep(portClaimInterval) + } +} + // FatalListenError explains a failed bind in plain language and stops the // process without a stack trace. It exits with ExitCodeFatalConfig so that a // service manager knows this is a configuration problem and does not restart diff --git a/version/restart.go b/version/restart.go new file mode 100644 index 0000000..440438e --- /dev/null +++ b/version/restart.go @@ -0,0 +1,266 @@ +// Copyright 2026 The casbin Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package version + +import ( + "encoding/json" + "fmt" + "os" + "path/filepath" + "runtime/debug" + "time" + + "github.com/apache/casbin-gateway/util" +) + +// An update leaves two files next to the executable. One says a restart into a +// new version is under way, so the Gateway it starts knows it is that version +// and what it can be rolled back to. The other says the last attempt failed, so +// the Gateway that comes back says why instead of leaving the reader with a +// dialog that never moves. +const ( + restartFileName = ".update-restart" + failureFileName = ".update-failure" + // failedSuffix names the executable a rollback set aside. It is deleted on + // the next start, once nothing runs from it. + failedSuffix = ".failed" + // failureMaxAge keeps a file written by an update nobody watched from + // reporting a failure at some unrelated start weeks later. + failureMaxAge = 10 * time.Minute +) + +type restartFile struct { + // Target is the version being installed, as the web UI shows it. + Target string `json:"target"` + // Backup is the executable this one replaced, and is where a rollback puts + // the installation back to. + Backup string `json:"backup"` +} + +type failureFile struct { + Target string `json:"target"` + Error string `json:"error"` + At time.Time `json:"at"` +} + +// startedBy is the update that started this process, or nil when it was started +// the ordinary way. +var startedBy *restartFile + +// BeginStartup takes over what an update left behind: whether this process is +// the version it installed, and whether the last attempt failed. +// +// It is also where the executable a previous update replaced is deleted - but +// not when this process is an update's new version, because that executable is +// what a failed start is put back from. +func BeginStartup() { + executable, err := executablePath() + if err != nil { + return + } + dir := filepath.Dir(executable) + + if failure := readFailure(dir); failure != nil { + setStatus(Status{Stage: StageFailed, Target: failure.Target, Error: failure.Error, RolledBack: true}) + } + + startedBy = readRestart(dir) + if startedBy == nil { + CleanupBackup() + return + } + + // The download is finished with either way; the backup is not. + _ = os.RemoveAll(filepath.Join(dir, stagingDir)) +} + +// FinishStartup is the update taking. This version is about to serve, so there +// is nothing left to roll back to. +func FinishStartup() { + if startedBy == nil { + return + } + + startedBy = nil + if executable, err := executablePath(); err == nil { + _ = os.Remove(filepath.Join(filepath.Dir(executable), restartFileName)) + } + CleanupBackup() +} + +// RollBackFailedStart puts back the executable an update replaced and starts it, +// so a version that cannot run does not leave the machine with no Gateway at +// all. It reports whether it did, and records why for the Gateway it starts to +// show in place of the update it was watching. +func RollBackFailedStart(reason string) bool { + if startedBy == nil { + return false + } + + previous := startedBy + startedBy = nil + + executable, err := executablePath() + if err != nil { + return false + } + if _, err := os.Stat(previous.Backup); err != nil { + return false + } + + // The executable of a running process can be renamed on every platform, and + // replaced in place on none, so the one that failed is moved aside first. + failed := executable + failedSuffix + _ = os.Remove(failed) + if err := os.Rename(executable, failed); err != nil { + fmt.Printf("Casbin Gateway: the version that could not start is still installed: %v\n", err) + return false + } + if err := os.Rename(previous.Backup, executable); err != nil { + _ = os.Rename(failed, executable) + fmt.Printf("Casbin Gateway: the previous version could not be put back: %v\n", err) + return false + } + + rollBackDesktopLauncher(filepath.Dir(executable)) + + dir := filepath.Dir(executable) + _ = os.Remove(filepath.Join(dir, restartFileName)) + writeFailure(dir, failureFile{Target: previous.Target, Error: reason, At: time.Now()}) + + fmt.Printf("Casbin Gateway: %s could not start (%s), the previous version was put back and is starting\n", previous.Target, reason) + + workingDir, err := os.Getwd() + if err != nil { + workingDir = dir + } + + statusLock.Lock() + daemonLog := logPath + statusLock.Unlock() + + if _, err := util.StartDetached(executable, workingDir, daemonLog); err != nil { + fmt.Printf("Casbin Gateway: the previous version is installed but did not start: %v\n", err) + } + + return true +} + +// RollBackOnPanic is deferred by a Gateway on its way up: a version that panics +// before it serves is one this machine cannot run, so the one it replaced is +// put back rather than left dead. An ordinary start has nothing to roll back +// to, so the panic is raised again as it would have been. +func RollBackOnPanic() { + crash := recover() + if crash == nil { + return + } + // Nothing to roll back to: the panic is raised again as it would have been, + // stack trace and all. + if startedBy == nil { + panic(crash) + } + + fmt.Printf("Casbin Gateway: %v\n%s\n", crash, debug.Stack()) + if !RollBackFailedStart(fmt.Sprintf("%v", crash)) { + panic(crash) + } +} + +// rollBackDesktopLauncher puts back the launcher the same update replaced. It +// is best-effort: an installation with no desktop has none, and a launcher left +// one version ahead of the server still starts it. +func rollBackDesktopLauncher(installDir string) { + launcher := filepath.Join(installDir, desktopLauncherName()) + backup := launcher + backupSuffix + if _, err := os.Stat(backup); err != nil { + return + } + + failed := launcher + failedSuffix + _ = os.Remove(failed) + if err := os.Rename(launcher, failed); err != nil { + return + } + if err := os.Rename(backup, launcher); err != nil { + _ = os.Rename(failed, launcher) + } +} + +// noteRestart records the update about to start a new version, and undoes the +// record when that version could not be started at all. +func noteRestart(executable string, target string) { + writeRestart(filepath.Dir(executable), restartFile{ + Target: target, + Backup: executable + backupSuffix, + }) +} + +func forgetRestart(executable string) { + _ = os.Remove(filepath.Join(filepath.Dir(executable), restartFileName)) +} + +func readRestart(dir string) *restartFile { + content, err := os.ReadFile(filepath.Join(dir, restartFileName)) + if err != nil { + return nil + } + + var file restartFile + if err := json.Unmarshal(content, &file); err != nil || file.Backup == "" { + return nil + } + + return &file +} + +func writeRestart(dir string, file restartFile) { + content, err := json.Marshal(file) + if err != nil { + return + } + + _ = os.WriteFile(filepath.Join(dir, restartFileName), content, 0o600) +} + +// readFailure takes the recorded failure and removes it, so it is reported to +// whoever is watching this start and never again. +func readFailure(dir string) *failureFile { + path := filepath.Join(dir, failureFileName) + content, err := os.ReadFile(path) + if err != nil { + return nil + } + _ = os.Remove(path) + + var file failureFile + if err := json.Unmarshal(content, &file); err != nil || file.Error == "" { + return nil + } + if time.Since(file.At) > failureMaxAge { + return nil + } + + return &file +} + +func writeFailure(dir string, file failureFile) { + content, err := json.Marshal(file) + if err != nil { + return + } + + _ = os.WriteFile(filepath.Join(dir, failureFileName), content, 0o600) +} diff --git a/version/update.go b/version/update.go index dd06fee..c570051 100644 --- a/version/update.go +++ b/version/update.go @@ -57,6 +57,9 @@ type Status struct { Error string `json:"error"` // Network means the failure was reaching GitHub, which a proxy can fix. Network bool `json:"network"` + // RolledBack means the version this update installed could not start and + // the one it replaced is what is running now. + RolledBack bool `json:"rolledBack"` } // stagingDir holds the download and the unpacked executable. It sits next to @@ -340,9 +343,15 @@ func restart(executable string) error { statusLock.Lock() daemonLog := logPath + target := status.Target statusLock.Unlock() + // What the new version needs to know about the one it replaced, which it + // puts back if it cannot serve. + noteRestart(executable, target) + if _, err := util.StartDetached(executable, workingDir, daemonLog); err != nil { + forgetRestart(executable) return fmt.Errorf("the new executable is installed but did not start: %w", err) } @@ -357,10 +366,10 @@ func restart(executable string) error { return nil } -// CleanupBackup removes what an update left behind. It runs at startup and -// keeps trying in the background for as long as the process it replaced takes -// to go away, rather than leaving a second copy of the executable on disk until -// the next restart. +// CleanupBackup removes what an update left behind, once there is nothing left +// to roll back to. It keeps trying in the background for as long as the process +// it replaced takes to go away, rather than leaving a second copy of the +// executable on disk until the next restart. func CleanupBackup() { executable, err := executablePath() if err != nil { @@ -369,8 +378,11 @@ func CleanupBackup() { _ = os.RemoveAll(filepath.Join(filepath.Dir(executable), stagingDir)) + launcher := filepath.Join(filepath.Dir(executable), desktopLauncherName()) removeWhenUnlocked(executable + backupSuffix) - removeWhenUnlocked(filepath.Join(filepath.Dir(executable), desktopLauncherName()+backupSuffix)) + removeWhenUnlocked(executable + failedSuffix) + removeWhenUnlocked(launcher + backupSuffix) + removeWhenUnlocked(launcher + failedSuffix) } // removeWhenUnlocked deletes a replaced executable once nothing runs from it, diff --git a/web/src/components/shared/version-panel.tsx b/web/src/components/shared/version-panel.tsx index fb802c4..d6277cc 100644 --- a/web/src/components/shared/version-panel.tsx +++ b/web/src/components/shared/version-panel.tsx @@ -94,6 +94,19 @@ function blockedHint(blocked: string) { } } +/** + * What a failed update says. A rollback is the one failure the reader has + * something to take from: the Gateway in front of them is the version it went + * back to, and the rest of the line is why the new one did not run. + */ +function failureText(status: UpdateStatus) { + if (!status.rolledBack) { + return status.error; + } + + return `${i18next.t("general:The new version did not start, so the previous one was put back")}: ${status.error}`; +} + function stageLabel(status: UpdateStatus | null, phase: Phase) { if (phase === "waiting") { return i18next.t("general:Restarting into the new version"); @@ -204,7 +217,7 @@ export function VersionPanel({ } setStatus(res.data); if (res.data.stage === "failed") { - setFailure(res.data.error); + setFailure(failureText(res.data)); setFailureNetwork(res.data.network); setPhase("idle"); } else if (res.data.stage === "restarting") { @@ -247,23 +260,29 @@ export function VersionPanel({ MiscBackend.getUpdateStatus() .then(res => { - if (stopped || res.status !== "ok" || !res.data) { - return; - } - if (res.data.stage === "failed") { - stopped = true; - window.clearInterval(timer); - setFailure(res.data.error); - setFailureNetwork(res.data.network); - setPhase("idle"); + if (stopped) { return; } - if (res.data.stage === "idle") { - stopped = true; - window.clearInterval(timer); - setPhase("done"); - window.setTimeout(() => window.location.reload(), 1200); + if (res.status === "ok" && res.data) { + // Still the Gateway being replaced: it answers until it goes. + if (res.data.stage !== "idle" && res.data.stage !== "failed") { + return; + } + if (res.data.stage === "failed") { + stopped = true; + window.clearInterval(timer); + setFailure(failureText(res.data)); + setFailureNetwork(res.data.network); + setPhase("idle"); + return; + } } + // An answer that is not a status at all — a session this Gateway does + // not know, say — is still the restarted one answering. + stopped = true; + window.clearInterval(timer); + setPhase("done"); + window.setTimeout(() => window.location.reload(), 1200); }) // The gap where nothing answers is the restart itself. .catch(() => undefined); diff --git a/web/src/locales/en/data.json b/web/src/locales/en/data.json index 37e4c07..8aed0fc 100644 --- a/web/src/locales/en/data.json +++ b/web/src/locales/en/data.json @@ -990,6 +990,7 @@ "Sure to delete {name} ?": "Sure to delete {name} ?", "Terminal": "Terminal", "The new version did not start, check the Gateway log": "The new version did not start, check the Gateway log", + "The new version did not start, so the previous one was put back": "The new version did not start, so the previous one was put back", "This Gateway cannot find its own executable, so the update has to be done by hand": "This Gateway cannot find its own executable, so the update has to be done by hand", "This executable sits where it cannot be replaced, so the update has to be done by hand": "This executable sits where it cannot be replaced, so the update has to be done by hand", "This is a read-only demo site!": "This is a read-only demo site!", diff --git a/web/src/locales/zh/data.json b/web/src/locales/zh/data.json index 7bbe8a1..8534184 100644 --- a/web/src/locales/zh/data.json +++ b/web/src/locales/zh/data.json @@ -1059,6 +1059,7 @@ "Sure to delete {name} ?": "确定删除{name}？", "Terminal": "终端", "The new version did not start, check the Gateway log": "新版本没有启动，请查看Gateway日志", + "The new version did not start, so the previous one was put back": "新版本没有启动，已回退到原来的版本", "This Gateway cannot find its own executable, so the update has to be done by hand": "Gateway找不到自己的可执行文件，无法替换，需要手动升级", "This executable sits where it cannot be replaced, so the update has to be done by hand": "当前可执行文件所在目录不可写，无法被替换，需要手动升级", "This is a read-only demo site!": "这是一个只读演示站点！", diff --git a/web/src/types.ts b/web/src/types.ts index ae47b27..c78f18b 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -1468,6 +1468,8 @@ export interface UpdateStatus { error: string; /** The failure was reaching GitHub, which a proxy can fix. */ network: boolean; + /** The new version could not start, so the one it replaced is running. */ + rolledBack: boolean; } export interface VersionInfo { --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
