using System; using System.Diagnostics; using System.IO; using System.Drawing; using System.Drawing.Imaging; using System.Threading; using System.Threading.Tasks; using DesertPaintCodex.Models; using DesertPaintCodex.Util; namespace DesertPaintCodex.Services { public class ReactionScannerService { private enum LogVerbosity { Low, Normal, High, Excessive } // Swatch is 301x20 solid color, 1px darker left/top, 1px black outer left/top, 1px light right/bottom, 1px bright right/bottom // top-right and bottom-left are bright instead of black // Color bars are 4x302 solid color, 2px border on all sides (darker left/top, lighter bottom/right private static readonly int[] SWATCH_HEIGHT = { 24, // tiny 24, // small 24, // medium 24, // large 24 // huge }; private static readonly int[] SWATCH_WIDTH = { 306, // tiny 306, // small 306, // medium 320, // large 350 // huge }; private static readonly int[] COLOR_BAR_WIDTH = { 306, // tiny 306, // small -- includes left and right borders 306, // medium 320, // large 350 // huge }; private static readonly int[] RED_BAR_SPACING = { 32, // tiny 32, // small 32, // medium 32, // large 32 // huge }; private static readonly int[] GREEN_BAR_SPACING = { 42, // tiny 42, // small 42, // medium 42, // large 42 // huge }; private static readonly int[] BLUE_BAR_SPACING = { 52, // tiny 52, // small 52, // medium 52, // large 52 // huge }; // width to test on ends of swatch (10% on either end) private static readonly int[] SWATCH_TEST_WIDTH = { 26, // tiny 26, // small 26, // medium 28, // large 31 // huge }; private static ReactionScannerService? s_instance; public static ReactionScannerService Instance => s_instance ??= new ReactionScannerService(); private int _swatchHeight = SWATCH_HEIGHT[(int)Constants.DefaultInterfaceSize]; private int _swatchWidth = SWATCH_WIDTH[(int)Constants.DefaultInterfaceSize]; private int _swatchTestWidth = SWATCH_TEST_WIDTH[(int)Constants.DefaultInterfaceSize]; private int _colorBarWidth = COLOR_BAR_WIDTH[(int)Constants.DefaultInterfaceSize]; private int _redBarSpacing = RED_BAR_SPACING[(int)Constants.DefaultInterfaceSize]; private int _greenBarSpacing = GREEN_BAR_SPACING[(int)Constants.DefaultInterfaceSize]; private int _blueBarSpacing = BLUE_BAR_SPACING[(int)Constants.DefaultInterfaceSize]; // Current Status public bool IsCaptured { get; private set; } private PaintColor _recordedColor = new PaintColor(); public PaintColor RecordedColor { get => _recordedColor; private set => _recordedColor.Set(value); } private int _pixelMultiplier = 1; private InterfaceSize _interfaceSize = Constants.DefaultInterfaceSize; private bool _firstRun = true; private int _lastSwatchX = -1; private int _lastSwatchY = -1; private int _screenX = 0; private int _screenY = 0; private Bitmap _targetBitmap; private LogVerbosity _logVerbosity = LogVerbosity.Normal; private CancellationTokenSource? _canceler; public StreamWriter? Log { get; set; } private void WriteLog(LogVerbosity verbosity, string format, params object[] args) { // Dumping this to Debug, for now. if (verbosity <= _logVerbosity) { Debug.WriteLine(format, args); } /* if ((Log != null) && (verbosity <= _logVerbosity)) { Log.WriteLine(format, args); } */ } public ReactionScannerService() { RefreshFromSettings(); Debug.Assert(_targetBitmap != null); } public void RefreshFromSettings() { SettingsService.Get(SettingKey.PixelMultiplier, out _pixelMultiplier, Constants.DefaultPixelMultiplier); SettingsService.Get(SettingKey.InterfaceSize, out int interfaceSize, (int)Constants.DefaultInterfaceSize); if (Enum.IsDefined(typeof(InterfaceSize), interfaceSize)) { _interfaceSize = (InterfaceSize)interfaceSize; } SettingsService.Get(SettingKey.ScreenX, out _screenX, Constants.DefaultScreenX); SettingsService.Get(SettingKey.ScreenY, out _screenY, Constants.DefaultScreenY); SettingsService.Get(SettingKey.ScreenWidth, out int screenWidth, Constants.DefaultScreenWidth); SettingsService.Get(SettingKey.ScreenHeight, out int screenHeight, Constants.DefaultScreenHeight); _targetBitmap = new Bitmap(screenWidth, screenHeight); UpdateSwatchSizes(); } private void UpdateSwatchSizes() { _swatchHeight = SWATCH_HEIGHT[(int)_interfaceSize] * _pixelMultiplier; _swatchWidth = SWATCH_WIDTH[(int)_interfaceSize] * _pixelMultiplier; _colorBarWidth = COLOR_BAR_WIDTH[(int)_interfaceSize] * _pixelMultiplier; _redBarSpacing = RED_BAR_SPACING[(int)_interfaceSize] * _pixelMultiplier; _greenBarSpacing = GREEN_BAR_SPACING[(int)_interfaceSize] * _pixelMultiplier; _blueBarSpacing = BLUE_BAR_SPACING[(int)_interfaceSize] * _pixelMultiplier; _swatchTestWidth = SWATCH_TEST_WIDTH[(int)_interfaceSize] * _pixelMultiplier; } public void CancelScan() { _canceler?.Cancel(); } private static class ColorMatcher { public static Color Color; public static bool IsMatch(Color otherColor) { return PixelColor.IsMatch(Color, otherColor); } } private bool IsPossibleSwatchSlice(Pixels pixels, int x, int y) { // 1.) Check if the top pixel is a dark pixel. if (!PixelColor.IsDark(pixels.ColorAt(x, y))) return false; // 2.) grab the swatch color 2 down from top border ColorMatcher.Color = pixels.ColorAt(x, y + 2); // Scan the column from 2 below the top to 3 above the bottom to ensure the color matches for (int i = 2; i < (_swatchHeight-3); ++i) { if (!pixels.DoesPixelMatch(x, y + i, ColorMatcher.IsMatch)) return false; } return true; } private bool IsPossibleSwatchUpperLeft(Pixels pixels, int x, int y) { bool result = true; int swatchSolidWidth = _swatchWidth - 4; int swatchSolidHeight = _swatchHeight - 5; // 2 top and 3 bottom pixels are slightly different colors int swatchSolidLeftX = x + 2; int swatchSolidTopY = y + 2; int swatchSolidRightX = swatchSolidLeftX + swatchSolidWidth - 1; int swatchSolidBottomY = swatchSolidTopY + swatchSolidHeight - 1; Color swatchColor = pixels.ColorAt(swatchSolidLeftX, swatchSolidTopY); ColorMatcher.Color = swatchColor; // Check the other 3 corners of the swatch size for color match Color testColor = Color.Black; bool upperRightResult = pixels.DoesPixelMatch(swatchSolidRightX, swatchSolidTopY, ColorMatcher.IsMatch); if (!upperRightResult) { testColor = pixels.ColorAt(swatchSolidRightX, swatchSolidTopY); WriteLog(LogVerbosity.Excessive, "Upper-right mismatch for {8}, {9} - found {0},{1},{2} at {3}, {4} expected {5},{6},{7}", testColor.R, testColor.G, testColor.B, swatchSolidRightX, swatchSolidTopY, swatchColor.R, swatchColor.G, swatchColor.B, x, y); } bool lowerLeftResult = pixels.DoesPixelMatch(swatchSolidLeftX, swatchSolidBottomY, ColorMatcher.IsMatch); if (!lowerLeftResult) { testColor = pixels.ColorAt(swatchSolidLeftX, swatchSolidBottomY); WriteLog(LogVerbosity.Excessive, "Lower-left mismatch for {8}, {9} - found {0},{1},{2} at {3}, {4} expected {5},{6},{7}", testColor.R, testColor.G, testColor.B, swatchSolidLeftX, swatchSolidBottomY, swatchColor.R, swatchColor.G, swatchColor.B, x, y); } bool lowerRightResult = pixels.DoesPixelMatch(swatchSolidRightX, swatchSolidBottomY, ColorMatcher.IsMatch); if (!lowerRightResult) { testColor = pixels.ColorAt(swatchSolidRightX, swatchSolidBottomY); WriteLog(LogVerbosity.Excessive, "Lower-right mismatch for {8}, {9} - found {0},{1},{2} at {3}, {4} expected {5},{6},{7}", testColor.R, testColor.G, testColor.B, swatchSolidRightX, swatchSolidBottomY, swatchColor.R, swatchColor.G, swatchColor.B, x, y); } result &= upperRightResult; result &= lowerLeftResult; result &= lowerRightResult; if (!result) { // Box corners test failed WriteLog(LogVerbosity.High, "Failed to find left edge for potential swatch of color {2}, {3}, {4} at {0}, {1}", x, y, swatchColor.R, swatchColor.G, swatchColor.B); return false; } // scan down the right and left sides for (int yOff = 1; yOff < (swatchSolidHeight - 1); ++yOff) { result &= pixels.DoesPixelMatch(swatchSolidLeftX, swatchSolidTopY + yOff, ColorMatcher.IsMatch); if (!result) { testColor = pixels.ColorAt(swatchSolidLeftX, swatchSolidTopY + yOff); break; } result &= pixels.DoesPixelMatch(swatchSolidRightX, swatchSolidTopY + yOff, ColorMatcher.IsMatch); if (!result) { testColor = pixels.ColorAt(swatchSolidRightX, swatchSolidTopY + yOff); break; } } if (!result) { WriteLog(LogVerbosity.Normal, "Failed to find left/right edges for potential swatch of color {2}, {3}, {4} at {0}, {1} [failed color = {5},{6},{7}]", x, y, swatchColor.R, swatchColor.G, swatchColor.B, testColor.R, testColor.G, testColor.B); return false; } for (int xOff = 1; xOff < (swatchSolidWidth - 1); ++xOff) { result &= pixels.DoesPixelMatch(swatchSolidLeftX + xOff, swatchSolidTopY, ColorMatcher.IsMatch); if (!result) { testColor = pixels.ColorAt(swatchSolidLeftX + xOff, swatchSolidTopY); break; } result &= pixels.DoesPixelMatch(swatchSolidLeftX + xOff, swatchSolidBottomY, ColorMatcher.IsMatch); if (!result) { testColor = pixels.ColorAt(swatchSolidLeftX + xOff, swatchSolidBottomY); break; } } if (!result) { WriteLog(LogVerbosity.Normal, "Failed to match upper/lower edges for potential swatch of color {2}, {3}, {4} at {0}, {1} [failed color = {5},{6},{7}]", x, y, swatchColor.R, swatchColor.G, swatchColor.B, testColor.R, testColor.G, testColor.B); return false; } // test the left edge for dark pixels -- the bottom-most pixel is bright now int i = 0; for (i = 1; result && i < _swatchHeight - 1; ++i) { result &= pixels.DoesPixelMatch(x, y + i, PixelColor.IsDark); } if (!result) { // No dark border on the left side WriteLog(LogVerbosity.Normal, "Failed to find left border for potential swatch of color {2}, {3}, {4} at {0}, {1}", x, y, swatchColor.R, swatchColor.G, swatchColor.B); return false; } // test the dark top border and for papyrus above and below the swatch bool borderError = false; int papyErrorCount = 0; for (i = 0; result && (i < _swatchWidth - 1); ++i) { bool isBorder = pixels.DoesPixelMatch(x + i, y, PixelColor.IsDark); result &= isBorder; if (!isBorder) { WriteLog(LogVerbosity.Normal, "Probable swatch at {0},{1} failed upper border test at {2},{3}", x, y, x + i, y); borderError = true; } // Checking along the top of the swatch for papyrus // The row just above is shaded, so check 2 above if (y > 1) { bool isPapyrus = pixels.DoesPixelMatch(x + i, y - 2, PixelColor.IsPapyrus); papyErrorCount += isPapyrus ? 0 : 1; } else { ++papyErrorCount; } // Checking along the bottom of the swatch for papyrus if (y < pixels.Height) { bool isPapyrus = pixels.DoesPixelMatch(x + i, y + _swatchHeight, PixelColor.IsPapyrus); papyErrorCount += isPapyrus ? 0 : 1; } else { ++papyErrorCount; } } result &= (papyErrorCount < (_swatchWidth / 20)); // allow up to 5% error rate checking for papy texture, because this seems to be inconsistent if (!result && ((i > (_swatchWidth*0.8)) || (papyErrorCount >= (_swatchWidth/20)))) { if (!borderError && (papyErrorCount < _swatchWidth)) { WriteLog(LogVerbosity.Normal, "Found a potential swatch candidate of width {0} at {1},{2} that had {3} failures matching papyrus texture", i, x, y, papyErrorCount); } } return result; } private bool TestPosition(int x, int y, Pixels pixels, ref PaintColor reactedColor) { // Check 4 corners of solid area and left/right solid bar areas bool foundSwatch = IsPossibleSwatchUpperLeft(pixels, x, y); // ((pixel_r < 0x46) && (pixel_g < 0x46) && (pixel_b < 0x46)); if (foundSwatch) { WriteLog(LogVerbosity.Normal, "Found probable swatch at {0},{1} - checking border slices", x, y); int borderXOffset = 0; for (borderXOffset = 2; foundSwatch && (borderXOffset < _swatchTestWidth); ++borderXOffset) { foundSwatch &= IsPossibleSwatchSlice(pixels, x + borderXOffset, y); if (!foundSwatch) { WriteLog(LogVerbosity.Normal, "Failed slice test at {0},{1}", x + borderXOffset, y); break; } foundSwatch &= IsPossibleSwatchSlice(pixels, x + _swatchWidth - borderXOffset, y); if (!foundSwatch) { WriteLog(LogVerbosity.Normal, "Failed slice test at {0},{1}", x + _swatchWidth - borderXOffset, y); break; } } } if (!foundSwatch) return false; // WE FOUND THE SWATCH! // Now we know where the color bars are. int redPixelCount = pixels.LengthOfColorAt(x, y + _redBarSpacing, PixelColor.IsRed); reactedColor.Red = (byte)Math.Round((float)redPixelCount * 255f / (float)_colorBarWidth); int greenPixelCount = pixels.LengthOfColorAt(x, y + _greenBarSpacing, PixelColor.IsGreen); reactedColor.Green = (byte)Math.Round((float)greenPixelCount * 255f / (float)_colorBarWidth); int bluePixelCount = pixels.LengthOfColorAt(x, y + _blueBarSpacing, PixelColor.IsBlue); reactedColor.Blue = (byte)Math.Round((float)bluePixelCount * 255f / (float)_colorBarWidth); WriteLog(LogVerbosity.Low, "Found the color swatch at {0}, {1}. Color={2} Red={3}px Green={4}px Blue={5}px", x, y, reactedColor, redPixelCount, greenPixelCount, bluePixelCount); return true; } public async Task CaptureReactionAsync(IProgress progress) { _canceler = new CancellationTokenSource(); // You can't re-use these, sadly. return await Task.Run(() => CaptureReaction(progress, _canceler.Token)); } private bool CaptureReaction(IProgress progress, CancellationToken cancellToken) { PaintColor reactedColor = new(); IsCaptured = false; _recordedColor.Clear(); using (var g = Graphics.FromImage(_targetBitmap)) { Debug.WriteLine("Scan starting at [" + _screenX + ", " + _screenY + "]"); g.CopyFromScreen(_screenX, _screenY, 0, 0, _targetBitmap.Size, CopyPixelOperation.SourceCopy); } _targetBitmap.Save(Path.Combine(ProfileManager.CurrentProfile?.Directory ?? "", "screenshot.png"), ImageFormat.Png); Pixels pixels = new(_targetBitmap, _pixelMultiplier); IsCaptured = false; if (!_firstRun) { // If this is not the first run, let's check the last location, to see if the UI is still there. if (TestPosition(_lastSwatchX, _lastSwatchY, pixels, ref reactedColor)) { IsCaptured = true; RecordedColor = reactedColor; return true; } _firstRun = true; } SettingsService.Get("Log.Verbosity", out var verbosityIdx, 1); _logVerbosity = (LogVerbosity)verbosityIdx; int patchTestSize = ((_swatchHeight - 5) / 2) - 1; SettingsService.Get("ScanArea.Min.X", out var startX, 0); SettingsService.Get("ScanArea.Min.Y", out var startY, 0); SettingsService.Get("ScanArea.Max.X", out var endX, _targetBitmap.Width); SettingsService.Get("ScanArea.Max.Y", out var endY, _targetBitmap.Height); startX = Math.Max(2, Math.Min(startX, _targetBitmap.Width - 2)); startY = Math.Max(2, Math.Min(startY, _targetBitmap.Height - 2)); endX = Math.Min(_targetBitmap.Width - 2, Math.Max(2, endX)) - _colorBarWidth; // + patchTestSize; endY = Math.Min(_targetBitmap.Height - 2, Math.Max(2, endY)) - (_blueBarSpacing + 10); // + patchTestSize; Debug.WriteLine("startX: " + startX + " endX: " + endX + " startY: " + startY + " endY: " + endY); int xSpan = endX - startX; int ySpan = endY - startY; int total = xSpan * ySpan; for (int roughX = startX; roughX < endX ; roughX += patchTestSize) { int xMark = roughX - startX; for (int roughY = startY; roughY < endY; roughY += patchTestSize) { progress.Report((float)((xMark * ySpan) + (roughY - startY)) / total); cancellToken.ThrowIfCancellationRequested(); if (!pixels.IsSolidPatchAt(roughX, roughY, patchTestSize, patchTestSize)) continue; Color patchColor = pixels.ColorAt(roughX, roughY); WriteLog(LogVerbosity.Excessive, "Found a solid patch of {2},{3},{4} at {0}, {1}", roughX, roughY, patchColor.R, patchColor.G, patchColor.B); for (int x = Math.Max(0, roughX - patchTestSize); x < roughX; ++x) { for (int y = Math.Max(0, roughY - patchTestSize); y < roughY; ++y) { WriteLog(LogVerbosity.Excessive, "Searching for potential swatch at {0},{1} after found square at {2},{3}", x, y, roughX, roughY); if (!TestPosition(x, y, pixels, ref reactedColor)) continue; RecordedColor = reactedColor; _lastSwatchX = x; _lastSwatchY = y; _firstRun = false; IsCaptured = true; return true; } } WriteLog(LogVerbosity.Excessive, "False-positive patch of color {0},{1},{2} at {3},{4}", patchColor.R, patchColor.G, patchColor.B, roughX, roughY); } } return false; } public static Reaction Calculate3WayReaction(PlayerProfile profile, PaintColor expectedColor, PaintColor reactedColor, Reagent reagent0, Reagent reagent1, Reagent reagent2) { // A 3-reagent reaction. Reaction? reaction1 = profile.FindReaction(reagent0, reagent1); Reaction? reaction2 = profile.FindReaction(reagent0, reagent2); Debug.Assert(reaction1 != null); Debug.Assert(reaction2 != null); int r = reactedColor.Red - expectedColor.Red - reaction1.Red - reaction2.Red; int g = reactedColor.Green - expectedColor.Green - reaction1.Green - reaction2.Green; int b = reactedColor.Blue - expectedColor.Blue - reaction1.Blue - reaction2.Blue; return new Reaction(r, g, b); } public static Reaction CalculateReaction(PaintColor expectedColor, PaintColor reactedColor) { // A 2-reagent reaction. int r = reactedColor.Red - expectedColor.Red; int g = reactedColor.Green - expectedColor.Green; int b = reactedColor.Blue - expectedColor.Blue; return new Reaction(r, g, b); } } }