diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs
index 088d77195..5cf22eabe 100644
--- a/GVFS/GVFS.Common/GVFSConstants.cs
+++ b/GVFS/GVFS.Common/GVFSConstants.cs
@@ -67,6 +67,20 @@ public static class GitConfig
public const string MountProgress = GVFSPrefix + "mount-progress";
public const bool MountProgressDefault = false;
+ /* Opt-in switch for NUL-delimited streaming of large "-z" git status/diff output.
+ * Default false: use the bounded-buffer path with its truncation fail-safes (the proven
+ * behavior). Set true to stream instead, which processes an arbitrarily large result
+ * without buffering it. Off by default per the feature-flag convention so the rollout
+ * infrastructure can enable streaming gradually. */
+ public const string StreamGitStatusOutput = GVFSPrefix + "stream-git-status-output";
+ public const bool StreamGitStatusOutputDefault = false;
+
+ /* Optional watchdog for the streaming status/diff read: kill the git process if it does
+ * not finish within this many seconds. Default -1 (infinite / disabled) so a legitimately
+ * long status on a very large working tree is never killed; operators can opt in. */
+ public const string GitStatusStreamTimeoutSeconds = GVFSPrefix + "git-status-stream-timeout-seconds";
+ public const int GitStatusStreamTimeoutSecondsDefault = -1;
+
public const string MaxHttpConnectionsConfig = GVFSPrefix + "max-http-connections";
public const string PrefetchUseIdx = GVFSPrefix + "prefetch-use-idx";
diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs
index 0b06604f0..f3100a90c 100644
--- a/GVFS/GVFS.Common/Git/GitProcess.cs
+++ b/GVFS/GVFS.Common/Git/GitProcess.cs
@@ -37,6 +37,16 @@ public class GitProcess : ICredentialStore
///
private const int MaxCapturedStdOutChars = 128 * 1024 * 1024; // ~256 MB of UTF-16
+ ///
+ /// Upper bound on how long the NUL-streaming path waits for a git process to exit after its
+ /// stdout has reached EOF, when the caller did not set a finite timeout. Once stdout closes the
+ /// process is normally about to exit, so this only matters in the pathological case where git
+ /// wedges or a timeout tree-kill only partially succeeds (a surviving grandchild still holding
+ /// the pipe). Bounding the wait guarantees the streaming path can never pin the calling thread
+ /// indefinitely - the maintenance/prefetch hang this streaming change is meant to avoid.
+ ///
+ private const int DefaultPostReadGraceMs = 60 * 1000;
+
private static readonly Encoding UTF8NoBOM = new UTF8Encoding(false);
private static bool failedToSetEncoding = false;
private static string expireTimeDateString;
@@ -534,6 +544,55 @@ public bool TryGetFromConfig(string settingName, bool forceOutsideEnlistment, ou
return false;
}
+ ///
+ /// Reads a boolean git-config value, returning when the setting is
+ /// unset or unreadable. Uses git's boolean semantics (true/yes/on/1 => true; false/no/off/0/empty
+ /// => false).
+ ///
+ public virtual bool GetConfigBoolOrDefault(string settingName, bool defaultValue)
+ {
+ if (this.TryGetFromConfig(settingName, forceOutsideEnlistment: false, out string value) && value != null)
+ {
+ switch (value.Trim().ToLowerInvariant())
+ {
+ case "true":
+ case "yes":
+ case "on":
+ case "1":
+ return true;
+ case "false":
+ case "no":
+ case "off":
+ case "0":
+ return false;
+ }
+ }
+
+ // Unset, unreadable, or an unrecognized value falls back to the (safe) default.
+ return defaultValue;
+ }
+
+ ///
+ /// Reads an integer git-config value, returning when the setting is
+ /// unset or unparseable.
+ ///
+ public virtual int GetConfigIntOrDefault(string settingName, int defaultValue)
+ {
+ try
+ {
+ ConfigResult result = this.GetFromConfig(settingName, forceOutsideEnlistment: false);
+ if (result.TryParseAsInt(defaultValue, int.MinValue, out int value, out string _))
+ {
+ return value;
+ }
+ }
+ catch
+ {
+ }
+
+ return defaultValue;
+ }
+
public ConfigResult GetOriginUrl()
{
/* Disable precommand hook because this config call is used during mounting process
@@ -577,16 +636,39 @@ public Result Status(bool allowObjectDownloads, bool useStatusCache, bool showUn
return this.InvokeGitInWorkingDirectoryRoot(command, useReadObjectHook: allowObjectDownloads);
}
+ ///
+ /// Buffers the entire "git status" porcelain -z output and returns it on .
+ /// Bounded by the stdout capture cap; check before acting on
+ /// the result. This is the fallback used when streaming is disabled via
+ /// .
+ ///
public Result StatusPorcelain()
{
- string command = "status -uall --porcelain -z";
- return this.InvokeGitInWorkingDirectoryRoot(command, useReadObjectHook: false);
+ return this.InvokeGitInWorkingDirectoryRoot(StatusPorcelainCommand, useReadObjectHook: false);
}
///
- /// Returns staged file changes (index vs HEAD) as null-separated pairs of
- /// status and path: "A\0path1\0M\0path2\0D\0path3\0".
- /// Status codes: A=added, M=modified, D=deleted, R=renamed, C=copied.
+ /// Streams "git status" output in porcelain -z form, delivering each NUL-terminated record to
+ /// as it is read. This avoids buffering the entire status
+ /// output, which can be large in a big working tree.
+ ///
+ /// Receives each NUL-terminated record as it is read.
+ ///
+ /// Watchdog timeout in milliseconds, or -1 () for
+ /// no bound. If positive and the read does not finish in time, the git process is killed and the
+ /// result reports failure.
+ ///
+ public Result StatusPorcelain(Action parseStdOutToken, int timeoutMs = -1)
+ {
+ return this.InvokeGitInWorkingDirectoryRoot(StatusPorcelainCommand, useReadObjectHook: false, parseStdOutToken: parseStdOutToken, timeoutMs: timeoutMs);
+ }
+
+ ///
+ /// Buffers staged file changes (index vs HEAD) as NUL-separated records and returns them on
+ /// in the form "A\0path1\0M\0path2\0...". Bounded by the stdout
+ /// capture cap; check before acting on the result. This is
+ /// the fallback used when streaming is disabled via
+ /// .
///
/// Inline pathspecs to scope the diff, or null for all.
///
@@ -598,6 +680,40 @@ public Result StatusPorcelain()
/// separated by NUL instead of newline (--pathspec-file-nul).
///
public Result DiffCachedNameStatus(string[] pathspecs = null, string pathspecFromFile = null, bool pathspecFileNul = false)
+ {
+ string command = DiffCachedNameStatusCommand(pathspecs, pathspecFromFile, pathspecFileNul);
+ return this.InvokeGitInWorkingDirectoryRoot(command, useReadObjectHook: false);
+ }
+
+ ///
+ /// Streams staged file changes (index vs HEAD) as NUL-separated records: each change is emitted
+ /// as two records, a status token ("A", "M", "D", ...) followed by a path token. The records are
+ /// delivered to as they are read, so an arbitrarily large
+ /// staged set is processed without buffering the whole list.
+ ///
+ /// Receives each NUL-terminated record (status, path, status, path, ...).
+ /// Inline pathspecs to scope the diff, or null for all.
+ ///
+ /// Path to a file containing additional pathspecs (one per line), forwarded
+ /// as --pathspec-from-file to git. Null if not used.
+ ///
+ ///
+ /// When true and pathspecFromFile is set, pathspec entries in the file are
+ /// separated by NUL instead of newline (--pathspec-file-nul).
+ ///
+ ///
+ /// Watchdog timeout in milliseconds, or -1 () for
+ /// no bound.
+ ///
+ public Result DiffCachedNameStatus(Action parseStdOutToken, string[] pathspecs = null, string pathspecFromFile = null, bool pathspecFileNul = false, int timeoutMs = -1)
+ {
+ string command = DiffCachedNameStatusCommand(pathspecs, pathspecFromFile, pathspecFileNul);
+ return this.InvokeGitInWorkingDirectoryRoot(command, useReadObjectHook: false, parseStdOutToken: parseStdOutToken, timeoutMs: timeoutMs);
+ }
+
+ private const string StatusPorcelainCommand = "status -uall --porcelain -z";
+
+ private static string DiffCachedNameStatusCommand(string[] pathspecs, string pathspecFromFile, bool pathspecFileNul)
{
string command = "diff --cached --name-status -z --no-renames";
@@ -615,7 +731,7 @@ public Result DiffCachedNameStatus(string[] pathspecs = null, string pathspecFro
command += " -- " + string.Join(" ", pathspecs.Select(p => QuoteGitPath(p)));
}
- return this.InvokeGitInWorkingDirectoryRoot(command, useReadObjectHook: false);
+ return command;
}
///
@@ -995,13 +1111,22 @@ protected virtual Result InvokeGitImpl(
Action parseStdOutLine,
int timeoutMs,
string gitObjectsDirectory = null,
- bool usePreCommandHook = true)
+ bool usePreCommandHook = true,
+ Action parseStdOutToken = null)
{
if (failedToSetEncoding && writeStdIn != null)
{
return new Result(string.Empty, "Attempting to use to stdin, but the process does not have the right input encodings set.", Result.GenericFailureCode);
}
+ // NUL-delimited streaming reads stdout synchronously on this thread, so it cannot be combined
+ // with line streaming. A finite timeout is honored via a watchdog (see the streaming branch
+ // below) rather than the WaitForExit(timeoutMs) path used for buffered reads.
+ if (parseStdOutToken != null && parseStdOutLine != null)
+ {
+ throw new InvalidOperationException($"{nameof(parseStdOutToken)} cannot be combined with {nameof(parseStdOutLine)}.");
+ }
+
try
{
// From https://msdn.microsoft.com/en-us/library/system.diagnostics.process.standardoutput.aspx
@@ -1024,20 +1149,26 @@ protected virtual Result InvokeGitImpl(
errors.AppendLine(args.Data);
}
};
- this.executingProcess.OutputDataReceived += (sender, args) =>
+
+ // In NUL-delimited streaming mode we read stdout ourselves (below) rather than using
+ // the line-based async reader, so we do not subscribe OutputDataReceived.
+ if (parseStdOutToken == null)
{
- if (args.Data != null)
+ this.executingProcess.OutputDataReceived += (sender, args) =>
{
- if (parseStdOutLine != null)
- {
- parseStdOutLine(args.Data);
- }
- else
+ if (args.Data != null)
{
- output.AppendLine(args.Data);
+ if (parseStdOutLine != null)
+ {
+ parseStdOutLine(args.Data);
+ }
+ else
+ {
+ output.AppendLine(args.Data);
+ }
}
- }
- };
+ };
+ }
lock (this.executionLock)
{
@@ -1066,14 +1197,124 @@ protected virtual Result InvokeGitImpl(
writeStdIn?.Invoke(this.executingProcess.StandardInput);
this.executingProcess.StandardInput.Close();
- this.executingProcess.BeginOutputReadLine();
+ // Always drain stderr asynchronously so the child can never block writing to it.
this.executingProcess.BeginErrorReadLine();
- if (!this.executingProcess.WaitForExit(timeoutMs))
+ if (parseStdOutToken != null)
+ {
+ // Read stdout synchronously, splitting on NUL and handing each record to the
+ // callback as it arrives. Because stderr is drained asynchronously above, a
+ // synchronous stdout read cannot deadlock. Only a single record is held in
+ // memory at a time, so an arbitrarily large result (e.g. every staged file in
+ // a monorepo) is processed without buffering the whole thing.
+ //
+ // Optional watchdog: the synchronous read would otherwise block forever on a
+ // git that never closes stdout. When a finite timeout is configured, arm a
+ // timer that kills the process tree; the blocked Read then returns EOF and we
+ // surface a timeout. Default (timeoutMs == Timeout.Infinite) leaves streaming
+ // unbounded, matching the buffered path.
+ bool killedByTimeout = false;
+ bool readCompleted = false;
+ Timer watchdog = null;
+ if (timeoutMs != Timeout.Infinite)
+ {
+ watchdog = new Timer(
+ _ =>
+ {
+ lock (this.processLock)
+ {
+ // Only kill if the read is still in progress. Guarding on
+ // readCompleted (set under the same lock once the read returns)
+ // prevents a late callback from reporting a false timeout or
+ // killing a subsequent process reused on this instance.
+ if (!readCompleted && this.executingProcess != null)
+ {
+ killedByTimeout = true;
+ GVFSPlatform.Instance.TryKillProcessTree(this.executingProcess.Id, out int _, out string _);
+ }
+ }
+ },
+ state: null,
+ dueTime: timeoutMs,
+ period: Timeout.Infinite);
+ }
+
+ try
+ {
+ ReadStdOutTokens(this.executingProcess.StandardOutput, parseStdOutToken);
+ }
+ catch
+ {
+ // The stdout read or a streaming callback threw. The child git process is
+ // still running; disposing the Process wrapper (the using block below) would
+ // not end the child, leaking it. Kill the process tree before letting the
+ // exception propagate. Do not set 'stopping' here (unlike
+ // TryKillRunningProcess): this instance may be reused for later git calls.
+ lock (this.processLock)
+ {
+ if (this.executingProcess != null)
+ {
+ GVFSPlatform.Instance.TryKillProcessTree(this.executingProcess.Id, out int _, out string _);
+ }
+ }
+
+ throw;
+ }
+ finally
+ {
+ // Disarm the watchdog under the lock so an in-flight callback either ran
+ // before this or becomes a no-op, then dispose the timer.
+ lock (this.processLock)
+ {
+ readCompleted = true;
+ }
+
+ watchdog?.Dispose();
+ }
+
+ // stdout is at EOF, so git is normally about to exit. Bound this final wait:
+ // neither the watchdog (now disposed) nor a caller timeout covers it, so a
+ // wedged git or a partially-successful tree-kill (a surviving grandchild still
+ // holding the pipe) would otherwise pin this thread forever - the exact
+ // maintenance/prefetch hang this change is meant to avoid. Reuse the caller's
+ // timeout as the grace when one was set, otherwise a generous fixed ceiling.
+ int postReadGraceMs = timeoutMs != Timeout.Infinite ? timeoutMs : DefaultPostReadGraceMs;
+ if (!this.WaitForExitWithCancellation(postReadGraceMs, out bool _))
+ {
+ // git is still alive after stdout closed. Kill the tree again and fail
+ // rather than trust a partial exit code or block indefinitely.
+ lock (this.processLock)
+ {
+ if (this.executingProcess != null)
+ {
+ GVFSPlatform.Instance.TryKillProcessTree(this.executingProcess.Id, out int _, out string _);
+ }
+ }
+
+ // Distinct, self-identifying message so a field occurrence is measurable
+ // through the caller's error logging (GitProcess has no tracer of its own).
+ return new Result(string.Empty, "GitProcess streaming read: git did not exit within " + postReadGraceMs + "ms after stdout closed; killed process tree. " + errors.ToString(), Result.GenericFailureCode, outputTruncated: false, errorsTruncated: errors.Truncated);
+ }
+
+ // git has exited, so this parameterless WaitForExit() is now instant and
+ // guarantees the async stderr readers have flushed before we read Errors.
+ this.executingProcess.WaitForExit();
+
+ if (killedByTimeout)
+ {
+ return new Result(string.Empty, "Operation timed out: " + errors.ToString(), Result.GenericFailureCode, outputTruncated: false, errorsTruncated: errors.Truncated);
+ }
+ }
+ else
{
- this.executingProcess.Kill();
+ this.executingProcess.BeginOutputReadLine();
+
+ if (!this.executingProcess.WaitForExit(timeoutMs))
+ {
+ this.executingProcess.Kill();
- return new Result(output.ToString(), "Operation timed out: " + errors.ToString(), Result.GenericFailureCode, output.Truncated, errors.Truncated);
+ return new Result(output.ToString(), "Operation timed out: " + errors.ToString(), Result.GenericFailureCode, output.Truncated, errors.Truncated);
+ }
}
}
@@ -1090,11 +1331,95 @@ protected virtual Result InvokeGitImpl(
}
}
+ ///
+ /// Waits up to (or indefinitely when negative) for
+ /// to exit. When can be
+ /// canceled, polls at a short interval so a cancellation (e.g. mount shutdown) is observed
+ /// promptly, since has no cancellation-aware overload;
+ /// otherwise it waits in a single call. Shared by the streaming and buffered paths so both bound
+ /// their waits identically.
+ ///
+ /// True if the process exited; false if it timed out or was canceled (the caller should kill it).
+ private bool WaitForExitWithCancellation(int timeoutMs, out bool cancellationRequested, CancellationToken cancellationToken = default)
+ {
+ cancellationRequested = false;
+
+ if (!cancellationToken.CanBeCanceled)
+ {
+ // No cancellation to observe, so a single bounded wait suffices - no need to poll.
+ return this.executingProcess.WaitForExit(timeoutMs);
+ }
+
+ const int PollIntervalMs = 100;
+ Stopwatch stopwatch = Stopwatch.StartNew();
+ while (true)
+ {
+ int waitMs = PollIntervalMs;
+ if (timeoutMs >= 0)
+ {
+ long remainingMs = timeoutMs - stopwatch.ElapsedMilliseconds;
+ if (remainingMs <= 0)
+ {
+ return false;
+ }
+
+ waitMs = (int)Math.Min(PollIntervalMs, remainingMs);
+ }
+
+ if (this.executingProcess.WaitForExit(waitMs))
+ {
+ return true;
+ }
+
+ if (cancellationToken.IsCancellationRequested)
+ {
+ cancellationRequested = true;
+ return false;
+ }
+ }
+ }
+
private static string GenerateCredentialVerbCommand(string verb)
{
return $"-c {GitConfigSetting.CredentialUseHttpPath}=true credential {verb}";
}
+ ///
+ /// Reads a redirected stdout stream that is NUL-delimited (git's "-z" machine-readable format),
+ /// invoking once per NUL-terminated record as it is read.
+ /// Only a single record is accumulated at a time, so an arbitrarily large result is processed
+ /// without buffering the entire stream.
+ ///
+ internal static void ReadStdOutTokens(StreamReader reader, Action parseStdOutToken)
+ {
+ StringBuilder token = new StringBuilder();
+ char[] buffer = new char[8192];
+ int read;
+
+ while ((read = reader.Read(buffer, 0, buffer.Length)) > 0)
+ {
+ for (int i = 0; i < read; i++)
+ {
+ if (buffer[i] == '\0')
+ {
+ parseStdOutToken(token.ToString());
+ token.Clear();
+ }
+ else
+ {
+ token.Append(buffer[i]);
+ }
+ }
+ }
+
+ // git's -z output always terminates the final record with a NUL, so there should be nothing
+ // left here. Flush any trailing partial record defensively rather than dropping it.
+ if (token.Length > 0)
+ {
+ parseStdOutToken(token.ToString());
+ }
+ }
+
private static string ParseValue(string contents, string prefix)
{
int startIndex = contents.IndexOf(prefix) + prefix.Length;
@@ -1233,7 +1558,9 @@ private Result InvokeGitInWorkingDirectoryRoot(
string command,
bool useReadObjectHook,
Action writeStdIn = null,
- Action parseStdOutLine = null)
+ Action parseStdOutLine = null,
+ Action parseStdOutToken = null,
+ int timeoutMs = -1)
{
return this.InvokeGitImpl(
command,
@@ -1242,7 +1569,8 @@ private Result InvokeGitInWorkingDirectoryRoot(
useReadObjectHook: useReadObjectHook,
writeStdIn: writeStdIn,
parseStdOutLine: parseStdOutLine,
- timeoutMs: -1);
+ timeoutMs: timeoutMs,
+ parseStdOutToken: parseStdOutToken);
}
///
diff --git a/GVFS/GVFS.UnitTests/Git/GitProcessTests.cs b/GVFS/GVFS.UnitTests/Git/GitProcessTests.cs
index 66b9ee656..31d6fe2c8 100644
--- a/GVFS/GVFS.UnitTests/Git/GitProcessTests.cs
+++ b/GVFS/GVFS.UnitTests/Git/GitProcessTests.cs
@@ -3,13 +3,144 @@
using GVFS.UnitTests.Mock.Common;
using GVFS.UnitTests.Mock.Git;
using NUnit.Framework;
+using System.Collections.Generic;
using System.Diagnostics;
+using System.IO;
+using System.Text;
namespace GVFS.UnitTests.Git
{
[TestFixture]
public class GitProcessTests
{
+ [TestCase]
+ public void ReadStdOutTokens_SplitsOnNul()
+ {
+ List tokens = ReadTokens("a.txt\0d/b.txt\0d/c.txt\0");
+ tokens.ShouldMatchInOrder("a.txt", "d/b.txt", "d/c.txt");
+ }
+
+ [TestCase]
+ public void ReadStdOutTokens_EmptyInputYieldsNoTokens()
+ {
+ ReadTokens(string.Empty).Count.ShouldEqual(0);
+ }
+
+ [TestCase]
+ public void ReadStdOutTokens_SingleNulYieldsOneEmptyRecord()
+ {
+ // A lone NUL is one zero-length record, not "no records". A caller pairing status/path
+ // records relies on this so its state machine does not silently swallow the separator.
+ List tokens = ReadTokens("\0");
+ tokens.Count.ShouldEqual(1);
+ tokens[0].ShouldEqual(string.Empty);
+ }
+
+ [TestCase]
+ public void ReadStdOutTokens_PreservesEmptyRecords()
+ {
+ // diff --name-status -z emits status and path as separate records; an empty record must
+ // still be delivered so a caller's status/path state machine stays aligned.
+ List tokens = ReadTokens("A\0path\0\0after-empty\0");
+ tokens.ShouldMatchInOrder("A", "path", string.Empty, "after-empty");
+ }
+
+ [TestCase]
+ public void ReadStdOutTokens_FlushesTrailingRecordWithoutNul()
+ {
+ List tokens = ReadTokens("a.txt\0trailing");
+ tokens.ShouldMatchInOrder("a.txt", "trailing");
+ }
+
+ [TestCase]
+ public void ReadStdOutTokens_ReassemblesRecordSpanningReadBoundary()
+ {
+ // A single record longer than the internal 8192-char read buffer must be reassembled across
+ // multiple reads rather than split.
+ string longPath = new string('x', 20000);
+ List tokens = ReadTokens("short\0" + longPath + "\0");
+
+ tokens.Count.ShouldEqual(2);
+ tokens[0].ShouldEqual("short");
+ tokens[1].ShouldEqual(longPath);
+ }
+
+ [TestCase]
+ public void DiffCachedNameStatus_StreamsRecordsAsTokens()
+ {
+ MockGitProcess git = new MockGitProcess();
+ git.SetExpectedCommandResult(
+ "diff --cached --name-status -z --no-renames",
+ () => new GitProcess.Result("A\0added.txt\0M\0modified.txt\0", string.Empty, GitProcess.Result.SuccessCode));
+
+ List tokens = new List();
+ GitProcess.Result result = git.DiffCachedNameStatus(t => tokens.Add(t));
+
+ result.ExitCodeIsSuccess.ShouldBeTrue();
+ tokens.ShouldMatchInOrder("A", "added.txt", "M", "modified.txt");
+
+ // Streaming mode delivers all data through the callback; Output is empty (production never
+ // subscribes OutputDataReceived when streaming), so callers cannot depend on Output here.
+ result.Output.ShouldEqual(string.Empty);
+ }
+
+ [TestCase]
+ public void StatusPorcelain_StreamsRecordsAsTokens()
+ {
+ MockGitProcess git = new MockGitProcess();
+ git.SetExpectedCommandResult(
+ "status -uall --porcelain -z",
+ () => new GitProcess.Result("A added.txt\0 M modified.txt\0", string.Empty, GitProcess.Result.SuccessCode));
+
+ List tokens = new List();
+ GitProcess.Result result = git.StatusPorcelain(t => tokens.Add(t));
+
+ result.ExitCodeIsSuccess.ShouldBeTrue();
+ tokens.ShouldMatchInOrder("A added.txt", " M modified.txt");
+ }
+
+ [TestCase]
+ public void DiffCachedNameStatus_BufferedFallbackReturnsOutput()
+ {
+ // With streaming disabled (gvfs.stream-git-status-output=false) callers use the buffered
+ // overload, which returns the whole -z blob on Result.Output for the caller to split.
+ MockGitProcess git = new MockGitProcess();
+ git.SetExpectedCommandResult(
+ "diff --cached --name-status -z --no-renames",
+ () => new GitProcess.Result("A\0added.txt\0M\0modified.txt\0", string.Empty, GitProcess.Result.SuccessCode));
+
+ GitProcess.Result result = git.DiffCachedNameStatus();
+
+ result.ExitCodeIsSuccess.ShouldBeTrue();
+ result.Output.ShouldEqual("A\0added.txt\0M\0modified.txt\0");
+ }
+
+ [TestCase]
+ public void StatusPorcelain_BufferedFallbackReturnsOutput()
+ {
+ MockGitProcess git = new MockGitProcess();
+ git.SetExpectedCommandResult(
+ "status -uall --porcelain -z",
+ () => new GitProcess.Result("A added.txt\0 M modified.txt\0", string.Empty, GitProcess.Result.SuccessCode));
+
+ GitProcess.Result result = git.StatusPorcelain();
+
+ result.ExitCodeIsSuccess.ShouldBeTrue();
+ result.Output.ShouldEqual("A added.txt\0 M modified.txt\0");
+ }
+
+ private static List ReadTokens(string content)
+ {
+ List tokens = new List();
+ using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(content)))
+ using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
+ {
+ GitProcess.ReadStdOutTokens(reader, token => tokens.Add(token));
+ }
+
+ return tokens;
+ }
+
[TestCase]
public void Init_PinsObjectFormatToSha1()
{
diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs
index 10da74bb0..593ea05aa 100644
--- a/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs
+++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitProcess.cs
@@ -93,7 +93,8 @@ protected override Result InvokeGitImpl(
Action parseStdOutLine,
int timeoutMs,
string gitObjectsDirectory = null,
- bool usePrecommandHook = true)
+ bool usePrecommandHook = true,
+ Action parseStdOutToken = null)
{
this.CommandsRun.Add(command);
this.DotGitDirectoriesUsed.Add(dotGitDirectory);
@@ -132,6 +133,22 @@ protected override Result InvokeGitImpl(
}
/* Future: result.Output should be set to null in this case */
}
+
+ if (parseStdOutToken != null && !string.IsNullOrEmpty(result.Output))
+ {
+ // Feed the mock output through the real production tokenizer so the test double cannot
+ // drift from ReadStdOutTokens' actual semantics (empty records, trailing-fragment flush).
+ using (MemoryStream stream = new MemoryStream(Encoding.UTF8.GetBytes(result.Output)))
+ using (StreamReader reader = new StreamReader(stream, Encoding.UTF8))
+ {
+ GitProcess.ReadStdOutTokens(reader, parseStdOutToken);
+ }
+
+ // In streaming mode production never subscribes OutputDataReceived, so Result.Output is
+ // empty; mirror that here so callers cannot rely on Output being populated after streaming.
+ result = new Result(string.Empty, result.Errors, result.ExitCode, result.OutputTruncated, result.ErrorsTruncated);
+ }
+
return result;
}
diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs
index a51b713d1..eaccc0c98 100644
--- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs
+++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs
@@ -455,54 +455,135 @@ public bool AddStagedFilesToModifiedPaths(string messageBody, out int addedCount
}
}
- // Query all staged files in one call using --name-status -z.
- // Output format: "A\0path1\0M\0path2\0D\0path3\0"
- GitProcess.Result result = gitProcess.DiffCachedNameStatus(pathspecs, pathspecFromFile, pathspecFileNul);
-
- if (result.OutputTruncated)
+ // Query all staged files in one call using --name-status -z. Records arrive in pairs: a
+ // status token ("A", "M", "D", ...) followed by a path token. By default we stream the
+ // records so we never buffer the entire (potentially huge) staged file list in memory; the
+ // gvfs.stream-git-status-output=false kill switch restores the bounded-capture path with its
+ // truncation fail-safe.
+ List addedFilePaths = new List();
+ int added = 0;
+
+ // Record a single (status, path) pair into ModifiedPaths / the hydration list. Shared by the
+ // streaming and buffered paths so both behave identically per record.
+ Action handleRecord = (status, gitPath) =>
{
- // The staged-file list exceeded the capture buffer. Acting on a partial list would leave
- // some staged files out of ModifiedPaths (skip-worktree not cleared, stale placeholders),
- // which is worse than failing. Fail safe and let the caller retry.
- EventMetadata metadata = new EventMetadata();
- metadata.Add("ExitCode", result.ExitCode);
- this.context.Tracer.RelatedError(
- metadata,
- nameof(this.AddStagedFilesToModifiedPaths) + ": git diff --cached output was truncated; refusing to update ModifiedPaths from a partial staged-file list");
- return false;
- }
+ if (string.IsNullOrEmpty(gitPath))
+ {
+ return;
+ }
- if (result.ExitCodeIsSuccess && !string.IsNullOrEmpty(result.Output))
- {
- string[] parts = result.Output.Split(new[] { '\0' }, StringSplitOptions.RemoveEmptyEntries);
- List addedFilePaths = new List();
+ string platformPath = gitPath.Replace(GVFSConstants.GitPathSeparator, Path.DirectorySeparatorChar);
+ if (this.modifiedPaths.TryAdd(platformPath, isFolder: false, isRetryable: out _))
+ {
+ added++;
+ }
- // Parts alternate: status, path, status, path, ...
- for (int i = 0; i + 1 < parts.Length; i += 2)
+ // Added files (in index but not in HEAD) are ProjFS placeholders that
+ // would vanish when the projection reverts to HEAD. Collect them for
+ // hydration below.
+ if (status.StartsWith("A"))
{
- string status = parts[i];
- string gitPath = parts[i + 1];
+ addedFilePaths.Add(gitPath);
+ }
+ };
+
+ bool streamOutput = gitProcess.GetConfigBoolOrDefault(
+ GVFSConstants.GitConfig.StreamGitStatusOutput,
+ GVFSConstants.GitConfig.StreamGitStatusOutputDefault);
- if (string.IsNullOrEmpty(gitPath))
+ GitProcess.Result result;
+ if (streamOutput)
+ {
+ int seconds = gitProcess.GetConfigIntOrDefault(
+ GVFSConstants.GitConfig.GitStatusStreamTimeoutSeconds,
+ GVFSConstants.GitConfig.GitStatusStreamTimeoutSecondsDefault);
+ int timeoutMs = (seconds > 0 && seconds <= int.MaxValue / 1000) ? seconds * 1000 : -1;
+
+ // Collect the (status, path) records as they stream in, but do NOT mutate ModifiedPaths
+ // yet: the callback runs while git is still executing and before the exit code is known,
+ // so applying records eagerly would leave a partial ModifiedPaths update behind if git
+ // later fails. Buffer the parsed records instead and apply them only after success below.
+ // Holding the records as many small strings (rather than git's whole stdout blob in one
+ // contiguous buffer) still avoids the large-array allocation that caused the OOM.
+ List> pendingRecords = new List>();
+ string pendingStatus = null;
+ result = gitProcess.DiffCachedNameStatus(
+ token =>
{
- continue;
- }
+ if (pendingStatus == null)
+ {
+ pendingStatus = token;
+ return;
+ }
- string platformPath = gitPath.Replace(GVFSConstants.GitPathSeparator, Path.DirectorySeparatorChar);
- if (this.modifiedPaths.TryAdd(platformPath, isFolder: false, isRetryable: out _))
+ string status = pendingStatus;
+ pendingStatus = null;
+ pendingRecords.Add(new KeyValuePair(status, token));
+ },
+ pathspecs,
+ pathspecFromFile,
+ pathspecFileNul,
+ timeoutMs);
+
+ if (result.ExitCodeIsSuccess && pendingStatus != null)
+ {
+ // The -z stream ended on a status token with no matching path, so the staged-file
+ // list is incomplete (e.g. git was killed mid-write). Acting on a partial list would
+ // leave staged files out of ModifiedPaths, so fail and let the caller retry rather
+ // than silently dropping the last entry.
+ EventMetadata incompleteMetadata = new EventMetadata();
+ incompleteMetadata.Add("ExitCode", result.ExitCode);
+ this.context.Tracer.RelatedError(
+ incompleteMetadata,
+ nameof(this.AddStagedFilesToModifiedPaths) + ": git diff --cached output ended on an unpaired status token; refusing to act on an incomplete staged-file list");
+ return false;
+ }
+
+ // Apply the buffered records only now that git exited successfully, so a mid-stream
+ // failure never leaves a partial ModifiedPaths mutation behind.
+ if (result.ExitCodeIsSuccess)
+ {
+ foreach (KeyValuePair record in pendingRecords)
{
- addedCount++;
+ handleRecord(record.Key, record.Value);
}
+ }
+
+ addedCount = added;
+ }
+ else
+ {
+ result = gitProcess.DiffCachedNameStatus(pathspecs, pathspecFromFile, pathspecFileNul);
- // Added files (in index but not in HEAD) are ProjFS placeholders that
- // would vanish when the projection reverts to HEAD. Collect them for
- // hydration below.
- if (status.StartsWith("A"))
+ if (result.OutputTruncated)
+ {
+ // The staged-file list exceeded the capture buffer. Acting on a partial list would
+ // leave some staged files out of ModifiedPaths (skip-worktree not cleared, stale
+ // placeholders), which is worse than failing. Fail safe and let the caller retry.
+ EventMetadata truncatedMetadata = new EventMetadata();
+ truncatedMetadata.Add("ExitCode", result.ExitCode);
+ this.context.Tracer.RelatedError(
+ truncatedMetadata,
+ nameof(this.AddStagedFilesToModifiedPaths) + ": git diff --cached output was truncated; refusing to update ModifiedPaths from a partial staged-file list");
+ return false;
+ }
+
+ if (result.ExitCodeIsSuccess && !string.IsNullOrEmpty(result.Output))
+ {
+ string[] parts = result.Output.Split(new[] { '\0' }, StringSplitOptions.RemoveEmptyEntries);
+
+ // Parts alternate: status, path, status, path, ...
+ for (int i = 0; i + 1 < parts.Length; i += 2)
{
- addedFilePaths.Add(gitPath);
+ handleRecord(parts[i], parts[i + 1]);
}
}
+ addedCount = added;
+ }
+
+ if (result.ExitCodeIsSuccess)
+ {
// Write added files from the git object store to disk as full files
// so they persist across projection changes. Batched into as few git
// process invocations as possible.
@@ -514,7 +595,7 @@ public bool AddStagedFilesToModifiedPaths(string messageBody, out int addedCount
}
}
}
- else if (!result.ExitCodeIsSuccess)
+ else
{
EventMetadata metadata = new EventMetadata();
metadata.Add("ExitCode", result.ExitCode);
diff --git a/GVFS/GVFS/CommandLine/SparseVerb.cs b/GVFS/GVFS/CommandLine/SparseVerb.cs
index 334e4d1ca..d1cc76de9 100644
--- a/GVFS/GVFS/CommandLine/SparseVerb.cs
+++ b/GVFS/GVFS/CommandLine/SparseVerb.cs
@@ -18,8 +18,8 @@ public class SparseVerb : GVFSVerb.ForExistingEnlistment
{
private const string SparseVerbName = "sparse";
private const string FolderListSeparator = ";";
- private const char StatusPathSeparatorToken = '\0';
private const char StatusRenameToken = 'R';
+ private const char StatusPathSeparatorToken = '\0';
private const string PruneOptionName = "prune";
private enum SetDirectoryTimeResult
@@ -628,29 +628,84 @@ private void ForceProjectionChange(ITracer tracer, GVFSEnlistment enlistment)
private void CheckGitStatus(ITracer tracer, GVFSEnlistment enlistment, HashSet sparseFolders)
{
GitProcess.Result statusResult = null;
- HashSet dirtyPathsNotInSparseSet = null;
+ HashSet dirtyPathsNotInSparseSet = new HashSet();
if (!this.ShowStatusWhileRunning(
() =>
{
+ dirtyPathsNotInSparseSet.Clear();
GitProcess git = new GitProcess(enlistment);
- statusResult = git.StatusPorcelain();
- if (statusResult.ExitCodeIsFailure)
+
+ bool streamOutput = git.GetConfigBoolOrDefault(
+ GVFSConstants.GitConfig.StreamGitStatusOutput,
+ GVFSConstants.GitConfig.StreamGitStatusOutputDefault);
+
+ if (streamOutput)
{
- return false;
+ int seconds = git.GetConfigIntOrDefault(
+ GVFSConstants.GitConfig.GitStatusStreamTimeoutSeconds,
+ GVFSConstants.GitConfig.GitStatusStreamTimeoutSecondsDefault);
+ int timeoutMs = (seconds > 0 && seconds <= int.MaxValue / 1000) ? seconds * 1000 : -1;
+
+ // Stream porcelain -z records so we never buffer the whole status output. Each entry
+ // is a primary "XY " token; a rename adds a second token for the original path.
+ bool expectingRenameOrigin = false;
+ statusResult = git.StatusPorcelain(
+ token =>
+ {
+ string gitPath;
+ if (expectingRenameOrigin)
+ {
+ expectingRenameOrigin = false;
+ gitPath = token;
+ }
+ else
+ {
+ if (token.Length < 3)
+ {
+ return;
+ }
+
+ // Two status chars (XY) then a space, then the path.
+ expectingRenameOrigin = token[0] == StatusRenameToken || token[1] == StatusRenameToken;
+ gitPath = token.Substring(3);
+ }
+
+ if (!PathCoveredBySparseFolders(gitPath, sparseFolders))
+ {
+ dirtyPathsNotInSparseSet.Add(gitPath);
+ }
+ },
+ timeoutMs);
+
+ if (statusResult.ExitCodeIsFailure)
+ {
+ return false;
+ }
}
-
- if (statusResult.OutputTruncated)
+ else
{
- // git status output exceeded the capture buffer. A partial status could omit
- // dirty paths and let sparse proceed over uncommitted changes (data loss), so
- // treat truncation as "cannot verify clean" and abort.
- tracer.RelatedError(
- new EventMetadata(),
- "git status output was truncated; aborting sparse to avoid acting on an incomplete status");
- return false;
+ // Buffered fallback (gvfs.stream-git-status-output=false): capture the whole status
+ // output, refuse to act on a truncated result, then parse it.
+ statusResult = git.StatusPorcelain();
+ if (statusResult.ExitCodeIsFailure)
+ {
+ return false;
+ }
+
+ if (statusResult.OutputTruncated)
+ {
+ // git status output exceeded the capture buffer. A partial status could omit
+ // dirty paths and let sparse proceed over uncommitted changes (data loss), so
+ // treat truncation as "cannot verify clean" and abort.
+ tracer.RelatedError(
+ new EventMetadata(),
+ "git status output was truncated; aborting sparse to avoid acting on an incomplete status");
+ return false;
+ }
+
+ dirtyPathsNotInSparseSet.UnionWith(this.GetPathsNotCoveredBySparseFolders(statusResult.Output, sparseFolders));
}
- dirtyPathsNotInSparseSet = this.GetPathsNotCoveredBySparseFolders(statusResult.Output, sparseFolders);
return dirtyPathsNotInSparseSet.Count == 0;
},
"Running git status",