Upgrade Scintilla from 4.1.4 to 4.2.0

Fix #5822

scintilla/boostregex/BoostRegExSearch.cxx

@@ -29,6 +29,7 @@
 #include "Decoration.h"
 #include "ILexer.h"
 #include "CaseFolder.h"
+#include "CharacterCategory.h"
 #include "Document.h"
 #include "UniConversion.h"
 #include "UTF8DocumentIterator.h"

scintilla/cocoa/InfoBar.mm

@@ -10,6 +10,8 @@
  * This file is dual licensed under LGPL v2.1 and the Scintilla license (http://www.scintilla.org/License.txt).
  */
 
+#include <cmath>
+
 #import "InfoBar.h"
 
 //--------------------------------------------------------------------------------------------------
@@ -33,7 +35,7 @@
 		CGFloat heightDelta = newRect.size.height - textSize.height;
 		if (heightDelta > 0) {
 			newRect.size.height -= heightDelta;
-			newRect.origin.y += ceil(heightDelta / 2);
+			newRect.origin.y += std::ceil(heightDelta / 2);
 		}
 	}
 
@@ -349,7 +351,7 @@ static float BarFontSize = 10.0;
 
 			// We only work with some preset zoom values. If the given value does not correspond
 			// to one then show no selection.
-			while (count < numberOfDefaultItems && (fabs(newScaleFactor - DefaultScaleMenuFactors[count]) > 0.07))
+			while (count < numberOfDefaultItems && (std::abs(newScaleFactor - DefaultScaleMenuFactors[count]) > 0.07))
 				count++;
 			if (count == numberOfDefaultItems)
 				[mZoomPopup selectItemAtIndex: -1];

scintilla/cocoa/PlatCocoa.h

@@ -80,7 +80,7 @@ public:
 	void PenColour(ColourDesired fore) override;
 
 	/** Returns a CGImageRef that represents the surface. Returns NULL if this is not possible. */
-	CGImageRef GetImage();
+	CGImageRef CreateImage();
 	void CopyImageRectangle(Surface &surfaceSource, PRectangle srcRect, PRectangle dstRect);
 
 	int LogPixelsY() override;

scintilla/cocoa/PlatCocoa.mm

@@ -17,6 +17,7 @@
 #include <cassert>
 #include <cstring>
 #include <cstdio>
+#include <cmath>
 
 #include <stdexcept>
 #include <string_view>
@@ -251,7 +252,7 @@ void AddToIntervalVector(std::vector<Interval> &vi, XYPOSITION left, XYPOSITION
 		vi.push_back(interval);
 	} else {
 		Interval &last = vi.back();
-		if (fabs(last.right-interval.left) < 0.01) {
+		if (std::abs(last.right-interval.left) < 0.01) {
 			// If new left is very close to previous right then extend last item
 			last.right = interval.right;
 		} else {
@@ -482,8 +483,8 @@ void SurfaceImpl::FillColour(const ColourDesired &back) {
 
 //--------------------------------------------------------------------------------------------------
 
-CGImageRef SurfaceImpl::GetImage() {
+CGImageRef SurfaceImpl::CreateImage() {
-	// For now, assume that GetImage can only be called on PixMap surfaces.
+	// For now, assume that CreateImage can only be called on PixMap surfaces.
 	if (!bitmapData)
 		return NULL;
 
@@ -628,8 +629,8 @@ void SurfaceImpl::FillRectangle(PRectangle rc, ColourDesired back) {
 	if (gc) {
 		FillColour(back);
 		// Snap rectangle boundaries to nearest int
-		rc.left = lround(rc.left);
+		rc.left = std::round(rc.left);
-		rc.right = lround(rc.right);
+		rc.right = std::round(rc.right);
 		CGRect rect = PRectangleToCGRect(rc);
 		CGContextFillRect(gc, rect);
 	}
@@ -654,7 +655,7 @@ void SurfaceImpl::FillRectangle(PRectangle rc, Surface &surfacePattern) {
 	SurfaceImpl &patternSurface = static_cast<SurfaceImpl &>(surfacePattern);
 
 	// For now, assume that copy can only be called on PixMap surfaces. Shows up black.
-	CGImageRef image = patternSurface.GetImage();
+	CGImageRef image = patternSurface.CreateImage();
 	if (image == NULL) {
 		FillRectangle(rc, ColourDesired(0));
 		return;
@@ -805,8 +806,8 @@ void Scintilla::SurfaceImpl::AlphaRectangle(PRectangle rc, int cornerSize, Colou
 		ColourDesired outline, int alphaOutline, int /*flags*/) {
 	if (gc) {
 		// Snap rectangle boundaries to nearest int
-		rc.left = lround(rc.left);
+		rc.left = std::round(rc.left);
-		rc.right = lround(rc.right);
+		rc.right = std::round(rc.right);
 		// Set the Fill color to match
 		CGContextSetRGBFillColor(gc, fill.GetRed() / 255.0, fill.GetGreen() / 255.0, fill.GetBlue() / 255.0, alphaFill / 255.0);
 		CGContextSetRGBStrokeColor(gc, outline.GetRed() / 255.0, outline.GetGreen() / 255.0, outline.GetBlue() / 255.0, alphaOutline / 255.0);
@@ -969,7 +970,7 @@ void SurfaceImpl::Ellipse(PRectangle rc, ColourDesired fore, ColourDesired back)
 
 void SurfaceImpl::CopyImageRectangle(Surface &surfaceSource, PRectangle srcRect, PRectangle dstRect) {
 	SurfaceImpl &source = static_cast<SurfaceImpl &>(surfaceSource);
-	CGImageRef image = source.GetImage();
+	CGImageRef image = source.CreateImage();
 
 	CGRect src = PRectangleToCGRect(srcRect);
 	CGRect dst = PRectangleToCGRect(dstRect);
@@ -1001,7 +1002,7 @@ void SurfaceImpl::Copy(PRectangle rc, Scintilla::Point from, Surface &surfaceSou
 	SurfaceImpl &source = static_cast<SurfaceImpl &>(surfaceSource);
 
 	// Get the CGImageRef
-	CGImageRef image = source.GetImage();
+	CGImageRef image = source.CreateImage();
 	// If we could not get an image reference, fill the rectangle black
 	if (image == NULL) {
 		FillRectangle(rc, ColourDesired(0));
@@ -1115,7 +1116,7 @@ void SurfaceImpl::DrawTextTransparent(PRectangle rc, Font &font_, XYPOSITION yba
 	CGColorRef color = CGColorCreateGenericRGB(colour.GetRed()/255.0, colour.GetGreen()/255.0, colour.GetBlue()/255.0, 1.0);
 
 	QuartzTextStyle *style = TextStyleFromFont(font_);
-	style->setCTStyleColor(color);
+	style->setCTStyleColour(color);
 
 	CGColorRelease(color);
 
@@ -1232,7 +1233,7 @@ XYPOSITION SurfaceImpl::AverageCharWidth(Font &font_) {
 
 	XYPOSITION width = WidthText(font_, sizeString);
 
-	return round(width / strlen(sizeString));
+	return std::round(width / strlen(sizeString));
 }
 
 void SurfaceImpl::SetClip(PRectangle rc) {
@@ -1470,7 +1471,7 @@ static NSImage *ImageFromXPM(XPM *pxpm) {
 		SurfaceImpl *surfaceIXPM = static_cast<SurfaceImpl *>(surfaceXPM.get());
 		CGContextClearRect(surfaceIXPM->GetContext(), CGRectMake(0, 0, width, height));
 		pxpm->Draw(surfaceXPM.get(), rcxpm);
-		CGImageRef imageRef = surfaceIXPM->GetImage();
+		CGImageRef imageRef = surfaceIXPM->CreateImage();
 		img = [[NSImage alloc] initWithCGImage: imageRef size: NSZeroSize];
 		CGImageRelease(imageRef);
 	}
@@ -1750,7 +1751,7 @@ void ListBoxImpl::SetFont(Font &font_) {
 	font.SetID(new QuartzTextStyle(*style));
 	NSFont *pfont = (__bridge NSFont *)style->getFontRef();
 	[colText.dataCell setFont: pfont];
-	CGFloat itemHeight = ceil(pfont.boundingRectForFont.size.height);
+	CGFloat itemHeight = std::ceil(pfont.boundingRectForFont.size.height);
 	table.rowHeight = itemHeight;
 }
 

scintilla/cocoa/QuartzTextStyle.h

@@ -47,8 +47,8 @@ public:
 		return styleDict;
 	}
 
-	void setCTStyleColor(CGColor *inColor) {
+	void setCTStyleColour(CGColor *inColour) {
-		CFDictionarySetValue(styleDict, kCTForegroundColorAttributeName, inColor);
+		CFDictionarySetValue(styleDict, kCTForegroundColorAttributeName, inColour);
 	}
 
 	float getAscent() const {

scintilla/cocoa/ScintillaCocoa.h

@@ -32,6 +32,7 @@
 #include "PropSetSimple.h"
 #endif
 
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"
@@ -196,7 +197,7 @@ public:
 	void ObserverRemove();
 	void IdleWork() override;
 	void QueueIdleWork(WorkNeeded::workItems items, Sci::Position upTo) override;
-	ptrdiff_t InsertText(NSString *input);
+	ptrdiff_t InsertText(NSString *input, CharacterSource charSource);
 	NSRange PositionsFromCharacters(NSRange rangeCharacters) const;
 	NSRange CharactersFromPositions(NSRange rangePositions) const;
 	NSString *RangeTextAsString(NSRange rangePositions) const;

scintilla/cocoa/ScintillaCocoa.mm

@@ -14,6 +14,8 @@
  * This file is dual licensed under LGPL v2.1 and the Scintilla license (http://www.scintilla.org/License.txt).
  */
 
+#include <cmath>
+
 #include <string_view>
 #include <vector>
 
@@ -316,7 +318,7 @@ const CGFloat paddingHighlightY = 2;
 		// main thread). We need that later for idle event processing.
 		NSNotificationCenter *center = [NSNotificationCenter defaultCenter];
 		notificationQueue = [[NSNotificationQueue alloc] initWithNotificationCenter: center];
-		[center addObserver: self selector: @selector(idleTriggered:) name: @"Idle" object: nil];
+		[center addObserver: self selector: @selector(idleTriggered:) name: @"Idle" object: self];
 	}
 	return self;
 }
@@ -818,7 +820,7 @@ namespace {
  */
 
 bool SupportAnimatedFind() {
-	return floor(NSAppKitVersionNumber) < NSAppKitVersionNumber10_12;
+	return std::floor(NSAppKitVersionNumber) < NSAppKitVersionNumber10_12;
 }
 
 }
@@ -1407,7 +1409,7 @@ void ScintillaCocoa::StartDrag() {
 	//           the full rectangle which may include non-selected text.
 
 	NSBitmapImageRep *bitmap = NULL;
-	CGImageRef imagePixmap = pixmap.GetImage();
+	CGImageRef imagePixmap = pixmap.CreateImage();
 	if (imagePixmap)
 		bitmap = [[NSBitmapImageRep alloc] initWithCGImage: imagePixmap];
 	CGImageRelease(imagePixmap);
@@ -2173,26 +2175,39 @@ bool ScintillaCocoa::KeyboardInput(NSEvent *event) {
 /**
  * Used to insert already processed text provided by the Cocoa text input system.
  */
-ptrdiff_t ScintillaCocoa::InsertText(NSString *input) {
+ptrdiff_t ScintillaCocoa::InsertText(NSString *input, CharacterSource charSource) {
-	CFStringEncoding encoding = EncodingFromCharacterSet(IsUnicodeMode(),
+	if ([input length] == 0) {
-				    vs.styles[STYLE_DEFAULT].characterSet);
+		return 0;
-	std::string encoded = EncodedBytesString((__bridge CFStringRef)input, encoding);
+	}
 
-	if (encoded.length() > 0) {
-		if (encoding == kCFStringEncodingUTF8) {
 	// There may be multiple characters in input so loop over them
+	if (IsUnicodeMode()) {
+		// There may be non-BMP characters as 2 elements in NSString so
+		// convert to UTF-8 and use UTF8BytesOfLead.
+		std::string encoded = EncodedBytesString((__bridge CFStringRef)input,
+							 kCFStringEncodingUTF8);
 		std::string_view sv = encoded;
 		while (sv.length()) {
 			const unsigned char leadByte = sv[0];
 			const unsigned int bytesInCharacter = UTF8BytesOfLead[leadByte];
-				AddCharUTF(sv.data(), bytesInCharacter, false);
+			InsertCharacter(sv.substr(0, bytesInCharacter), charSource);
 			sv.remove_prefix(bytesInCharacter);
 		}
-		} else {
-			AddCharUTF(encoded.c_str(), static_cast<unsigned int>(encoded.length()), false);
-		}
-	}
 		return encoded.length();
+	} else {
+		const CFStringEncoding encoding = EncodingFromCharacterSet(IsUnicodeMode(),
+									   vs.styles[STYLE_DEFAULT].characterSet);
+		ptrdiff_t lengthInserted = 0;
+		for (NSInteger i = 0; i < [input length]; i++) {
+			NSString *character = [input substringWithRange:NSMakeRange(i, 1)];
+			std::string encoded = EncodedBytesString((__bridge CFStringRef)character,
+								 encoding);
+			lengthInserted += encoded.length();
+			InsertCharacter(encoded, charSource);
+		}
+
+		return lengthInserted;
+	}
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -2272,7 +2287,7 @@ void ScintillaCocoa::CompositionStart() {
  */
 void ScintillaCocoa::CompositionCommit() {
 	pdoc->TentativeCommit();
-	pdoc->DecorationSetCurrentIndicator(INDIC_IME);
+	pdoc->DecorationSetCurrentIndicator(INDICATOR_IME);
 	pdoc->DecorationFillRange(0, 0, pdoc->Length());
 }
 
@@ -2486,7 +2501,7 @@ void ScintillaCocoa::ShowFindIndicatorForRange(NSRange charRange, BOOL retaining
 		layerFindIndicator = [[FindHighlightLayer alloc] init];
 		[content setWantsLayer: YES];
 		layerFindIndicator.geometryFlipped = content.layer.geometryFlipped;
-		if (floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_8) {
+		if (std::floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_8) {
 			// Content layer is unflipped on 10.9, but the indicator shows wrong unless flipped
 			layerFindIndicator.geometryFlipped = YES;
 		}

scintilla/cocoa/ScintillaFramework/Info.plist

@@ -17,11 +17,11 @@
 	<key>CFBundlePackageType</key>
 	<string>FMWK</string>
 	<key>CFBundleShortVersionString</key>
-	<string>4.1.4</string>
+	<string>4.2.0</string>
 	<key>CFBundleSignature</key>
 	<string>????</string>
 	<key>CFBundleVersion</key>
-	<string>4.1.4</string>
+	<string>4.2.0</string>
 	<key>NSPrincipalClass</key>
 	<string></string>
 </dict>

scintilla/cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj

@@ -7,6 +7,8 @@
 	objects = {
 
 /* Begin PBXBuildFile section */
+		00724A59981D34F11A3D162F /* LexCIL.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 577F46B88F633198B56D088D /* LexCIL.cxx */; };
+		0ED84236A703D57578EBFD2F /* LexNim.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 47814937A6B72D2B0F065B61 /* LexNim.cxx */; };
 		1100F1EB178E393200105727 /* CaseConvert.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 1100F1E6178E393200105727 /* CaseConvert.cxx */; };
 		1100F1EC178E393200105727 /* CaseConvert.h in Headers */ = {isa = PBXBuildFile; fileRef = 1100F1E7178E393200105727 /* CaseConvert.h */; };
 		1100F1ED178E393200105727 /* CaseFolder.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 1100F1E8178E393200105727 /* CaseFolder.cxx */; };
@@ -198,6 +200,7 @@
 		280056FC188DDD2C00F200AE /* StringCopy.h in Headers */ = {isa = PBXBuildFile; fileRef = 280056F9188DDD2C00F200AE /* StringCopy.h */; };
 		280056FD188DDD2C00F200AE /* SubStyles.h in Headers */ = {isa = PBXBuildFile; fileRef = 280056FA188DDD2C00F200AE /* SubStyles.h */; };
 		28064A05190F12E100E6E47F /* LexDMIS.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 28064A04190F12E100E6E47F /* LexDMIS.cxx */; };
+		281225362256DD2D00AFE50C /* UniqueString.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 281225352256DD2D00AFE50C /* UniqueString.cxx */; };
 		282D4A961F53FE270082E4D3 /* ILoader.h in Headers */ = {isa = PBXBuildFile; fileRef = 282D4A951F53FE270082E4D3 /* ILoader.h */; settings = {ATTRIBUTES = (Public, ); }; };
 		28804B2C1EEE232E00C0D154 /* DBCS.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 28804B2B1EEE232E00C0D154 /* DBCS.cxx */; };
 		28A067111A36B42600B4966A /* LexHex.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 28A067101A36B42600B4966A /* LexHex.cxx */; };
@@ -224,19 +227,18 @@
 		28D516DA1830FFCA0047C93D /* mac_cursor_flipped@2x.png in Resources */ = {isa = PBXBuildFile; fileRef = 28D516D71830FFCA0047C93D /* mac_cursor_flipped@2x.png */; };
 		28FDA42119B6967B00BE27D7 /* LexBibTeX.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 28FDA42019B6967B00BE27D7 /* LexBibTeX.cxx */; };
 		3D994BD7A5EAC4FA5B3CFBDF /* LexMaxima.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 29B042978C1F93EF42F9E4AB /* LexMaxima.cxx */; };
+		5F804AA6B60FE695863A39FE /* LexStata.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 7623427695416AB1270EE023 /* LexStata.cxx */; };
 		8DC2EF530486A6940098B216 /* InfoPlist.strings in Resources */ = {isa = PBXBuildFile; fileRef = 089C1666FE841158C02AAC07 /* InfoPlist.strings */; };
 		8DC2EF570486A6940098B216 /* Cocoa.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 1058C7B1FEA5585E11CA2CBB /* Cocoa.framework */; };
 		F437405F9F32C7DEFCA38C11 /* LexIndent.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 282E41F3B9E2BFEDD6A05BE7 /* LexIndent.cxx */; };
 		FDC7442CAD70B9A67EF1639D /* LexSAS.cxx in Sources */ = {isa = PBXBuildFile; fileRef = A95147A1AB7CADB00DAFE724 /* LexSAS.cxx */; };
-		5F804AA6B60FE695863A39FE /* LexStata.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 7623427695416AB1270EE023 /* LexStata.cxx */; };
+		AE894E1CB7328CAE5B2EF47E /* LexX12.cxx in Sources */ = {isa = PBXBuildFile; fileRef = ADA64364A443F3E3F02D294E /* LexX12.cxx */; };
-		0ED84236A703D57578EBFD2F /* LexNim.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 47814937A6B72D2B0F065B61 /* LexNim.cxx */; }; 
+		902B40FE926FE48538B168F1 /* LexDataflex.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 362E48F5A7F79598CB0B037D /* LexDataflex.cxx */; };
-		00724A59981D34F11A3D162F /* LexCIL.cxx in Sources */ = {isa = PBXBuildFile; fileRef = 577F46B88F633198B56D088D /* LexCIL.cxx */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
 		0867D69BFE84028FC02AAC07 /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = /System/Library/Frameworks/Foundation.framework; sourceTree = "<absolute>"; };
 		0867D6A5FE840307C02AAC07 /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = /System/Library/Frameworks/AppKit.framework; sourceTree = "<absolute>"; };
-		089C1667FE841158C02AAC07 /* English */ = {isa = PBXFileReference; fileEncoding = 10; lastKnownFileType = text.plist.strings; name = English; path = English.lproj/InfoPlist.strings; sourceTree = "<group>"; };
 		1058C7B1FEA5585E11CA2CBB /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = /System/Library/Frameworks/Cocoa.framework; sourceTree = "<absolute>"; };
 		1100F1E6178E393200105727 /* CaseConvert.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = CaseConvert.cxx; path = ../../src/CaseConvert.cxx; sourceTree = "<group>"; };
 		1100F1E7178E393200105727 /* CaseConvert.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = CaseConvert.h; path = ../../src/CaseConvert.h; sourceTree = "<group>"; };
@@ -429,6 +431,7 @@
 		280056F9188DDD2C00F200AE /* StringCopy.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = StringCopy.h; path = ../../lexlib/StringCopy.h; sourceTree = "<group>"; };
 		280056FA188DDD2C00F200AE /* SubStyles.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = SubStyles.h; path = ../../lexlib/SubStyles.h; sourceTree = "<group>"; };
 		28064A04190F12E100E6E47F /* LexDMIS.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexDMIS.cxx; path = ../../lexers/LexDMIS.cxx; sourceTree = "<group>"; };
+		281225352256DD2D00AFE50C /* UniqueString.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = UniqueString.cxx; path = ../../src/UniqueString.cxx; sourceTree = "<group>"; };
 		282D4A951F53FE270082E4D3 /* ILoader.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; name = ILoader.h; path = ../../include/ILoader.h; sourceTree = "<group>"; };
 		282E41F3B9E2BFEDD6A05BE7 /* LexIndent.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexIndent.cxx; path = ../../lexers/LexIndent.cxx; sourceTree = SOURCE_ROOT; };
 		28804B2B1EEE232E00C0D154 /* DBCS.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = DBCS.cxx; path = ../../src/DBCS.cxx; sourceTree = "<group>"; };
@@ -455,16 +458,19 @@
 		28D516D51830FFCA0047C93D /* info_bar_bg@2x.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "info_bar_bg@2x.png"; sourceTree = "<group>"; };
 		28D516D61830FFCA0047C93D /* mac_cursor_busy@2x.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "mac_cursor_busy@2x.png"; sourceTree = "<group>"; };
 		28D516D71830FFCA0047C93D /* mac_cursor_flipped@2x.png */ = {isa = PBXFileReference; lastKnownFileType = image.png; path = "mac_cursor_flipped@2x.png"; sourceTree = "<group>"; };
+		28F2653B224C30D7000CF4A3 /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = en.lproj/InfoPlist.strings; sourceTree = "<group>"; };
 		28FDA42019B6967B00BE27D7 /* LexBibTeX.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexBibTeX.cxx; path = ../../lexers/LexBibTeX.cxx; sourceTree = "<group>"; };
 		29B042978C1F93EF42F9E4AB /* LexMaxima.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexMaxima.cxx; path = ../../lexers/LexMaxima.cxx; sourceTree = SOURCE_ROOT; };
 		32DBCF5E0370ADEE00C91783 /* Scintilla_Prefix.pch */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = Scintilla_Prefix.pch; sourceTree = "<group>"; };
-		8DC2EF5A0486A6940098B216 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
-		8DC2EF5B0486A6940098B216 /* Scintilla.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Scintilla.framework; sourceTree = BUILT_PRODUCTS_DIR; };
-		D2F7E79907B2D74100F64583 /* CoreData.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreData.framework; path = /System/Library/Frameworks/CoreData.framework; sourceTree = "<absolute>"; };
-		A95147A1AB7CADB00DAFE724 /* LexSAS.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexSAS.cxx; path = ../../lexers/LexSAS.cxx; sourceTree = SOURCE_ROOT; };
-		7623427695416AB1270EE023 /* LexStata.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexStata.cxx; path = ../../lexers/LexStata.cxx; sourceTree = SOURCE_ROOT; };
 		47814937A6B72D2B0F065B61 /* LexNim.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexNim.cxx; path = ../../lexers/LexNim.cxx; sourceTree = SOURCE_ROOT; };
 		577F46B88F633198B56D088D /* LexCIL.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexCIL.cxx; path = ../../lexers/LexCIL.cxx; sourceTree = SOURCE_ROOT; };
+		7623427695416AB1270EE023 /* LexStata.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexStata.cxx; path = ../../lexers/LexStata.cxx; sourceTree = SOURCE_ROOT; };
+		8DC2EF5A0486A6940098B216 /* Info.plist */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
+		8DC2EF5B0486A6940098B216 /* Scintilla.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Scintilla.framework; sourceTree = BUILT_PRODUCTS_DIR; };
+		A95147A1AB7CADB00DAFE724 /* LexSAS.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexSAS.cxx; path = ../../lexers/LexSAS.cxx; sourceTree = SOURCE_ROOT; };
+		D2F7E79907B2D74100F64583 /* CoreData.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreData.framework; path = /System/Library/Frameworks/CoreData.framework; sourceTree = "<absolute>"; };
+		ADA64364A443F3E3F02D294E /* LexX12.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexX12.cxx; path = ../../lexers/LexX12.cxx; sourceTree = SOURCE_ROOT; };
+		362E48F5A7F79598CB0B037D /* LexDataflex.cxx */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; name = LexDataflex.cxx; path = ../../lexers/LexDataflex.cxx; sourceTree = SOURCE_ROOT; };
 /* End PBXFileReference section */
 
 /* Begin PBXFrameworksBuildPhase section */
@@ -590,6 +596,7 @@
 				114B6ED111FA7526004FB6AB /* LexCsound.cxx */,
 				114B6ED211FA7526004FB6AB /* LexCSS.cxx */,
 				114B6ED311FA7526004FB6AB /* LexD.cxx */,
+				362E48F5A7F79598CB0B037D /* LexDataflex.cxx */,
 				28B647071B54C0720009DC49 /* LexDiff.cxx */,
 				11FF3FE11810EB3900E13F13 /* LexDMAP.cxx */,
 				28064A04190F12E100E6E47F /* LexDMIS.cxx */,
@@ -671,6 +678,7 @@
 				114B6F0A11FA7526004FB6AB /* LexVerilog.cxx */,
 				114B6F0B11FA7526004FB6AB /* LexVHDL.cxx */,
 				11594BE8155B91DF0099E1FA /* LexVisualProlog.cxx */,
+				ADA64364A443F3E3F02D294E /* LexX12.cxx */,
 				114B6F0C11FA7526004FB6AB /* LexYAML.cxx */,
 			);
 			name = Lexers;
@@ -772,6 +780,7 @@
 				114B6F7311FA7598004FB6AB /* Style.cxx */,
 				114B6F9511FA75BE004FB6AB /* StyleContext.cxx */,
 				114B6F7411FA7598004FB6AB /* UniConversion.cxx */,
+				281225352256DD2D00AFE50C /* UniqueString.cxx */,
 				114B6F7511FA7598004FB6AB /* ViewStyle.cxx */,
 				114B6F9611FA75BE004FB6AB /* WordList.cxx */,
 				114B6F7611FA7598004FB6AB /* XPM.cxx */,
@@ -934,13 +943,11 @@
 			};
 			buildConfigurationList = 1DEB91B108733DA50010E9CD /* Build configuration list for PBXProject "ScintillaFramework" */;
 			compatibilityVersion = "Xcode 3.2";
-			developmentRegion = English;
+			developmentRegion = en;
 			hasScannedForEncodings = 1;
 			knownRegions = (
-				English,
+				Base,
-				Japanese,
+				en,
-				French,
-				German,
 			);
 			mainGroup = 0867D691FE84028FC02AAC07 /* Scintilla */;
 			productRefGroup = 034768DFFF38A50411DB9C8B /* Products */;
@@ -980,6 +987,7 @@
 				2744E5B60FC168C500E85C33 /* ScintillaView.mm in Sources */,
 				28D191A21DEA72C800159938 /* LexEDIFACT.cxx in Sources */,
 				28BDA1221EFB8F7C00EBD3F3 /* DefaultLexer.cxx in Sources */,
+				281225362256DD2D00AFE50C /* UniqueString.cxx in Sources */,
 				114B6F0D11FA7526004FB6AB /* LexAbaqus.cxx in Sources */,
 				114B6F0E11FA7526004FB6AB /* LexAda.cxx in Sources */,
 				28B6470C1B54C0720009DC49 /* LexBatch.cxx in Sources */,
@@ -1128,6 +1136,8 @@
 				5F804AA6B60FE695863A39FE /* LexStata.cxx in Sources */,
 				0ED84236A703D57578EBFD2F /* LexNim.cxx in Sources */,
 				00724A59981D34F11A3D162F /* LexCIL.cxx in Sources */,
+				AE894E1CB7328CAE5B2EF47E /* LexX12.cxx in Sources */,
+				902B40FE926FE48538B168F1 /* LexDataflex.cxx in Sources */,
 			);
 			runOnlyForDeploymentPostprocessing = 0;
 		};
@@ -1137,7 +1147,7 @@
 		089C1666FE841158C02AAC07 /* InfoPlist.strings */ = {
 			isa = PBXVariantGroup;
 			children = (
-				089C1667FE841158C02AAC07 /* English */,
+				28F2653B224C30D7000CF4A3 /* en */,
 			);
 			name = InfoPlist.strings;
 			sourceTree = "<group>";

scintilla/cocoa/ScintillaTest/ScintillaTest.xcodeproj/project.pbxproj

@@ -48,14 +48,14 @@
 /* End PBXCopyFilesBuildPhase section */
 
 /* Begin PBXFileReference section */
-		089C165DFE840E0CC02AAC07 /* English */ = {isa = PBXFileReference; fileEncoding = 10; lastKnownFileType = text.plist.strings; name = English; path = English.lproj/InfoPlist.strings; sourceTree = "<group>"; };
 		1058C7A1FEA54F0111CA2CBB /* Cocoa.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Cocoa.framework; path = /System/Library/Frameworks/Cocoa.framework; sourceTree = "<absolute>"; };
 		13E42FB307B3F0F600E4EEF1 /* CoreData.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreData.framework; path = /System/Library/Frameworks/CoreData.framework; sourceTree = "<absolute>"; };
-		1DDD58150DA1D0A300B32029 /* English */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = English; path = English.lproj/MainMenu.xib; sourceTree = "<group>"; };
 		271FA52A0F850BE20033D021 /* AppController.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = AppController.h; sourceTree = "<group>"; };
 		271FA52B0F850BE20033D021 /* AppController.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = AppController.mm; sourceTree = "<group>"; wrapsLines = 0; };
 		2744E5E20FC16BE200E85C33 /* ScintillaFramework.xcodeproj */ = {isa = PBXFileReference; lastKnownFileType = "wrapper.pb-project"; name = ScintillaFramework.xcodeproj; path = ../ScintillaFramework/ScintillaFramework.xcodeproj; sourceTree = SOURCE_ROOT; };
 		2791F4480FC1A8E9009DBCF9 /* TestData.sql */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text; path = TestData.sql; sourceTree = "<group>"; };
+		28E78A40224C33FE00456881 /* en */ = {isa = PBXFileReference; lastKnownFileType = text.plist.strings; name = en; path = en.lproj/InfoPlist.strings; sourceTree = "<group>"; };
+		28E78A41224C33FE00456881 /* en */ = {isa = PBXFileReference; lastKnownFileType = file.xib; name = en; path = en.lproj/MainMenu.xib; sourceTree = "<group>"; };
 		29B97316FDCFA39411CA2CEA /* main.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = "<group>"; };
 		29B97324FDCFA39411CA2CEA /* AppKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = AppKit.framework; path = /System/Library/Frameworks/AppKit.framework; sourceTree = "<absolute>"; };
 		29B97325FDCFA39411CA2CEA /* Foundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = Foundation.framework; path = /System/Library/Frameworks/Foundation.framework; sourceTree = "<absolute>"; };
@@ -195,13 +195,11 @@
 			};
 			buildConfigurationList = C01FCF4E08A954540054247B /* Build configuration list for PBXProject "ScintillaTest" */;
 			compatibilityVersion = "Xcode 3.2";
-			developmentRegion = English;
+			developmentRegion = en;
 			hasScannedForEncodings = 1;
 			knownRegions = (
-				English,
+				Base,
-				Japanese,
+				en,
-				French,
-				German,
 			);
 			mainGroup = 29B97314FDCFA39411CA2CEA /* ScintillaTest */;
 			projectDirPath = "";
@@ -265,7 +263,7 @@
 		089C165CFE840E0CC02AAC07 /* InfoPlist.strings */ = {
 			isa = PBXVariantGroup;
 			children = (
-				089C165DFE840E0CC02AAC07 /* English */,
+				28E78A40224C33FE00456881 /* en */,
 			);
 			name = InfoPlist.strings;
 			sourceTree = "<group>";
@@ -273,7 +271,7 @@
 		1DDD58140DA1D0A300B32029 /* MainMenu.xib */ = {
 			isa = PBXVariantGroup;
 			children = (
-				1DDD58150DA1D0A300B32029 /* English */,
+				28E78A41224C33FE00456881 /* en */,
 			);
 			name = MainMenu.xib;
 			sourceTree = "<group>";

scintilla/cocoa/ScintillaView.mm

@@ -9,6 +9,8 @@
  * This file is dual licensed under LGPL v2.1 and the Scintilla license (http://www.scintilla.org/License.txt).
  */
 
+#include <cmath>
+
 #include <string_view>
 #include <vector>
 
@@ -58,7 +60,7 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 	[super tile];
 
 #if defined(MAC_OS_X_VERSION_10_14)
-	if (floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_13) {
+	if (std::floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_13) {
 		NSRect frame = self.contentView.frame;
 		frame.origin.x = self.verticalRulerView.requiredThickness;
 		frame.size.width -= frame.origin.x;
@@ -413,7 +415,7 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 	}
 
 	[mOwner message: SCI_SETTARGETRANGE wParam: posRange.location lParam: NSMaxRange(posRange)];
-	std::string text([mOwner message: SCI_TARGETASUTF8] + 1, 0);
+	std::string text([mOwner message: SCI_TARGETASUTF8], 0);
 	[mOwner message: SCI_TARGETASUTF8 wParam: 0 lParam: reinterpret_cast<sptr_t>(&text[0])];
 	text = FixInvalidUTF8(text);
 	NSString *result = @(text.c_str());
@@ -423,12 +425,14 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 	// SCI_GETSTYLEAT reports a signed byte but want an unsigned to index into styles
 	const char styleByte = static_cast<char>([mOwner message: SCI_GETSTYLEAT wParam: posRange.location]);
 	const long style = static_cast<unsigned char>(styleByte);
-	std::string fontName([mOwner message: SCI_STYLEGETFONT wParam: style lParam: 0] + 1, 0);
+	std::string fontName([mOwner message: SCI_STYLEGETFONT wParam: style lParam: 0], 0);
 	[mOwner message: SCI_STYLEGETFONT wParam: style lParam: (sptr_t)&fontName[0]];
 	const CGFloat fontSize = [mOwner message: SCI_STYLEGETSIZEFRACTIONAL wParam: style] / 100.0f;
 	NSString *sFontName = @(fontName.c_str());
 	NSFont *font = [NSFont fontWithName: sFontName size: fontSize];
+	if (font) {
 		[asResult addAttribute: NSFontAttributeName value: font range: rangeAS];
+	}
 
 	return asResult;
 }
@@ -525,7 +529,7 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 	else if ([aString isKindOfClass: [NSAttributedString class]])
 		newText = (NSString *) [aString string];
 
-	mOwner.backend->InsertText(newText);
+	mOwner.backend->InsertText(newText, EditModel::CharacterSource::directInput);
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -618,11 +622,11 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 		NSRange posRangeCurrent = mOwner.backend->PositionsFromCharacters(NSMakeRange(replacementRange.location, 0));
 		// Note: Scintilla internally works almost always with bytes instead chars, so we need to take
 		//       this into account when determining selection ranges and such.
-		ptrdiff_t lengthInserted = mOwner.backend->InsertText(newText);
+		ptrdiff_t lengthInserted = mOwner.backend->InsertText(newText, EditModel::CharacterSource::tentativeInput);
 		posRangeCurrent.length = lengthInserted;
 		mMarkedTextRange = mOwner.backend->CharactersFromPositions(posRangeCurrent);
 		// Mark the just inserted text. Keep the marked range for later reset.
-		[mOwner setGeneralProperty: SCI_SETINDICATORCURRENT value: INDIC_IME];
+		[mOwner setGeneralProperty: SCI_SETINDICATORCURRENT value: INDICATOR_IME];
 		[mOwner setGeneralProperty: SCI_INDICATORFILLRANGE
 				 parameter: posRangeCurrent.location
 				     value: posRangeCurrent.length];
@@ -744,14 +748,14 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 		// Only snap for positions inside the document - allow outside
 		// for overshoot.
 		long lineHeight = mOwner.backend->WndProc(SCI_TEXTHEIGHT, 0, 0);
-		rc.origin.y = roundf(static_cast<XYPOSITION>(rc.origin.y) / lineHeight) * lineHeight;
+		rc.origin.y = std::round(static_cast<XYPOSITION>(rc.origin.y) / lineHeight) * lineHeight;
 	}
 	// Snap to whole points - on retina displays this avoids visual debris
 	// when scrolling horizontally.
 	if ((rc.origin.x > 0) && (NSMaxX(rc) < contentRect.size.width)) {
 		// Only snap for positions inside the document - allow outside
 		// for overshoot.
-		rc.origin.x = roundf(static_cast<XYPOSITION>(rc.origin.x));
+		rc.origin.x = std::round(static_cast<XYPOSITION>(rc.origin.x));
 	}
 	return rc;
 }
@@ -1233,7 +1237,7 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 #if MAC_OS_X_VERSION_MAX_ALLOWED > MAC_OS_X_VERSION_10_5
 	zoomDelta += event.magnification * 10.0;
 
-	if (fabs(zoomDelta)>=1.0) {
+	if (std::abs(zoomDelta)>=1.0) {
 		long zoomFactor = static_cast<long>([self getGeneralProperty: SCI_GETZOOM] + zoomDelta);
 		[self setGeneralProperty: SCI_SETZOOM parameter: zoomFactor value: 0];
 		zoomDelta = 0.0;
@@ -1372,11 +1376,11 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
  * input composition, depending on language, keyboard etc.
  */
 - (void) updateIndicatorIME {
-	[self setColorProperty: SCI_INDICSETFORE parameter: INDIC_IME fromHTML: @"#FF0000"];
+	[self setColorProperty: SCI_INDICSETFORE parameter: INDICATOR_IME fromHTML: @"#FF0000"];
 	const bool drawInBackground = [self message: SCI_GETPHASESDRAW] != 0;
-	[self setGeneralProperty: SCI_INDICSETUNDER parameter: INDIC_IME value: drawInBackground];
+	[self setGeneralProperty: SCI_INDICSETUNDER parameter: INDICATOR_IME value: drawInBackground];
-	[self setGeneralProperty: SCI_INDICSETSTYLE parameter: INDIC_IME value: INDIC_PLAIN];
+	[self setGeneralProperty: SCI_INDICSETSTYLE parameter: INDICATOR_IME value: INDIC_PLAIN];
-	[self setGeneralProperty: SCI_INDICSETALPHA parameter: INDIC_IME value: 100];
+	[self setGeneralProperty: SCI_INDICSETALPHA parameter: INDICATOR_IME value: 100];
 }
 
 //--------------------------------------------------------------------------------------------------
@@ -1398,7 +1402,7 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 #if defined(MAC_OS_X_VERSION_10_14)
 		// Let SCIScrollView account for other subviews such as vertical ruler by turning off
 		// automaticallyAdjustsContentInsets.
-		if (floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_13) {
+		if (std::floor(NSAppKitVersionNumber) > NSAppKitVersionNumber10_13) {
 			scrollView.contentView.automaticallyAdjustsContentInsets = NO;
 			scrollView.contentView.contentInsets = NSEdgeInsetsMake(0., 0., 0., 0.);
 		}
@@ -1961,9 +1965,9 @@ static NSCursor *cursorFromEnum(Window::Cursor cursor) {
 
 - (void) insertText: (id) aString {
 	if ([aString isKindOfClass: [NSString class]])
-		mBackend->InsertText(aString);
+		mBackend->InsertText(aString, EditModel::CharacterSource::directInput);
 	else if ([aString isKindOfClass: [NSAttributedString class]])
-		mBackend->InsertText([aString string]);
+		mBackend->InsertText([aString string], EditModel::CharacterSource::directInput);
 }
 
 //--------------------------------------------------------------------------------------------------

scintilla/doc/AddSource.txt

@@ -0,0 +1,30 @@
+Some of the build files adapt to adding and removing source code files but most
+must be modified by hand. Here is a list of directories and the build files that
+must be modified or possibly need to be modified.
+The Cocoa project.pbxproj file is complex and should be modified with Xcode.
+The other build files can be edited manually.
+
+src:
+	cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj
+	gtk/makefile
+	qt/ScintillaEdit/ScintillaEdit.pro
+	qt/ScintillaEditBase/ScintillaEditBase.pro
+	win32/makefile
+	win32/scintilla.mak
+	-- possibly:
+	test/unit/makefile
+	test/unit/test.mak
+
+cocoa:
+	cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj
+
+gtk:
+	gtk/makefile
+
+qt:
+	qt/ScintillaEdit/ScintillaEdit.pro
+	qt/ScintillaEditBase/ScintillaEditBase.pro
+
+win32:
+	win32/makefile
+	win32/scintilla.mak

scintilla/doc/Design.html

@@ -49,12 +49,12 @@
       library and with limited use of templates.
     </p>
     <p>
-       The currently supported platforms, Windows, GTK+/Linux, Cocoa and wxWidgets are fairly similar in
+       The currently supported platforms, Windows, GTK/Linux, Cocoa and wxWidgets are fairly similar in
       many ways.
       Each has windows, menus and bitmaps. These features generally work in similar ways so each
       has a way to move a window or draw a red line. Sometimes one platform requires a sequence of
       calls rather than a single call. At other times, the differences are more profound. Reading
-      the Windows clipboard occurs synchronously but reading the GTK+ clipboard requires a request
+      the Windows clipboard occurs synchronously but reading the GTK clipboard requires a request
       call that will be asynchronously answered with a message containing the clipboard data.
       The wxWidgets platform is available from the <a href="http://wxwidgets.org/">wxWidgets site</a>
     </p>
@@ -67,7 +67,7 @@
     </p>
     <p>
        The portability library is defined in Platform.h and is implemented once for each platform.
-      PlatWin.cxx defines the Windows variants of the methods and PlatGTK.cxx the GTK+ variants.
+      PlatWin.cxx defines the Windows variants of the methods and PlatGTK.cxx the GTK variants.
     </p>
     <p>
        Several of the classes here hold platform specific object identifiers and act as proxies to
@@ -83,7 +83,7 @@
     <p>
        These are simple classes provided to hold the commonly used geometric primitives. A
       PRectangle follows the Mac / Windows convention of not including its bottom and right sides
-      instead of including all its sides as is normal in GTK+. It is not called Rectangle as this may be
+      instead of including all its sides as is normal in GTK. It is not called Rectangle as this may be
       the name of a macro on Windows.
     </p>
     <h4>
@@ -101,7 +101,7 @@
        Font
     </h4>
     <p>
-       Font holds a platform specific font identifier - HFONT for Windows, PangoFontDescription* for GTK+. It
+       Font holds a platform specific font identifier - HFONT for Windows, PangoFontDescription* for GTK. It
       does not own the identifier and so will not delete the platform font object in its
       destructor. Client code should call Destroy at appropriate times.
     </p>
@@ -112,7 +112,7 @@
        Surface is an abstraction over each platform's concept of somewhere that graphical drawing
       operations can be done. It may wrap an already created drawing place such as a window or be
       used to create a bitmap that can be drawn into and later copied onto another surface. On
-      Windows it wraps a HDC and possibly a HBITMAP. On GTK+ it wraps a cairo_surface_t*.
+      Windows it wraps a HDC and possibly a HBITMAP. On GTK it wraps a cairo_surface_t*.
       Other platform specific objects are created (and correctly destroyed) whenever
       required to perform drawing actions.
     </p>
@@ -129,7 +129,7 @@
     <p>
        Window acts as a proxy to a platform window allowing operations such as showing, moving,
       redrawing, and destroying to be performed. It contains a platform specific window identifier
-      - HWND for Windows, GtkWidget* for GTK+.
+      - HWND for Windows, GtkWidget* for GTK.
     </p>
     <h4>
        ListBox
@@ -143,7 +143,7 @@
     </h4>
     <p>
        Menu is a small helper class for constructing popup menus. It contains the platform specific
-      menu identifier - HMENU for Windows, GtkMenu* for GTK+. Most of the work in
+      menu identifier - HMENU for Windows, GtkMenu* for GTK. Most of the work in
       constructing menus requires access to platform events and so is done in the Platform Events
       and API layer.
     </p>
@@ -224,20 +224,20 @@
     </h3>
     <p>
        Each platform uses different mechanisms for receiving events. On Windows, events are
-      received through messages and COM. On GTK+, callback functions are used.
+      received through messages and COM. On GTK, callback functions are used.
     </p>
     <p>
        For each platform, a class is derived from ScintillaBase (and thus from Editor). This is
-      ScintillaWin on Windows and ScintillaGTK on GTK+. These classes are responsible for
+      ScintillaWin on Windows and ScintillaGTK on GTK. These classes are responsible for
       connecting to the platforms event mechanism and also to implement some virtual methods in
       Editor and ScintillaBase which are different on the platforms. For example, this layer has to
-      support this difference between the synchronous Windows clipboard and the asynchronous GTK+
+      support this difference between the synchronous Windows clipboard and the asynchronous GTK
       clipboard.
     </p>
     <p>
        The external API is defined in this layer as each platform has different preferred styles of
-      API - messages on Windows and function calls on GTK+. This also allows multiple APIs to be
+      API - messages on Windows and function calls on GTK. This also allows multiple APIs to be
-      defined on a platform. The currently available API on GTK+ is similar to the Windows API and
+      defined on a platform. The currently available API on GTK is similar to the Windows API and
       does not follow platform conventions well. A second API could be implemented here that did
       follow platform conventions.
     </p>

scintilla/doc/ScintillaDownload.html

@@ -26,10 +26,10 @@
     <table bgcolor="#CCCCCC" width="100%" cellspacing="0" cellpadding="8" border="0">
       <tr>
         <td>
-          <font size="4"> <a href="https://www.scintilla.org/scintilla414.zip">
+          <font size="4"> <a href="https://www.scintilla.org/scintilla420.zip">
 	Windows</a>&nbsp;&nbsp;
-	<a href="https://www.scintilla.org/scintilla414.tgz">
+	<a href="https://www.scintilla.org/scintilla420.tgz">
-          GTK+/Linux</a>&nbsp;&nbsp;
+          GTK/Linux</a>&nbsp;&nbsp;
 	</font>
         </td>
       </tr>
@@ -42,7 +42,7 @@
       containing very few restrictions.
     </p>
     <h3>
-       Release 4.1.4
+       Release 4.2.0
     </h3>
     <h4>
        Source Code
@@ -50,8 +50,8 @@
        The source code package contains all of the source code for Scintilla but no binary
 	executable code and is available in
        <ul>
-       <li><a href="https://www.scintilla.org/scintilla414.zip">zip format</a> (1700K) commonly used on Windows</li>
+       <li><a href="https://www.scintilla.org/scintilla420.zip">zip format</a> (1.7M) commonly used on Windows</li>
-       <li><a href="https://www.scintilla.org/scintilla414.tgz">tgz format</a> (1500K) commonly used on Linux and compatible operating systems</li>
+       <li><a href="https://www.scintilla.org/scintilla420.tgz">tgz format</a> (1.4M) commonly used on Linux and compatible operating systems</li>
        </ul>
        Instructions for building on both Windows and Linux are included in the readme file.
     <h4>

scintilla/doc/ScintillaHistory.html

@@ -541,10 +541,14 @@
       </tr><tr>
 	<td>Jad Altahan</td>
 	<td>Andrea Ricchi</td>
+	<td>Juarez Rudsatz</td>
+	<td>Wil van Antwerpen</td>
+      </tr><tr>
+	<td>Hodong Kim</td>
     </tr>
     </table>
     <p>
-       Images used in GTK+ version
+       Images used in GTK version
     </p>
     <ul>
       <li>
@@ -552,6 +556,173 @@
         Icons</a> Copyright(C) 1998 by Dean S. Jones<br />
       </li>
     </ul>
+    <h3>
+       <a href="https://www.scintilla.org/scite420.zip">Release 4.2.0</a>
+    </h3>
+    <ul>
+	<li>
+	Released 5 July 2019.
+	</li>
+ 	<li>
+	Scintilla.iface adds line and pointer types, increases use of the position type, uses enumeration
+	types in methods and properties, and adds enumeration aliases to produce better CamelCase
+	identifiers.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1297/">Feature #1297</a>.
+	</li>
+ 	<li>
+	Source of input (direct / IME composition / IME result) reported in SCN_CHARADDED so applications
+	can treat temporary IME composition input differently.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2038/">Bug #2038</a>.
+	</li>
+ 	<li>
+	Lexer added for DataFlex.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1295/">Feature #1295</a>.
+	</li>
+ 	<li>
+	Matlab lexer now treats keywords as case-sensitive.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2112/">Bug #2112</a>.
+	</li>
+ 	<li>
+	SQL lexer fixes single quoted strings where '" (quote, double quote) was seen as continuing the string.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2098/">Bug #2098</a>.
+	</li>
+ 	<li>
+	Platform layers should use InsertCharacter method to perform keyboard and IME input, replacing
+	AddCharUTF method.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1293/">Feature #1293</a>.
+	</li>
+ 	<li>
+	Add CARETSTYLE_BLOCK_AFTER option to always display block caret after selection.
+	<a href="https://sourceforge.net/p/scintilla/bugs/1924/">Bug #1924</a>.
+	</li>
+ 	<li>
+	On Win32, limit text returned from WM_GETTEXT to the length specified in wParam.
+	This could cause failures when using assistive technologies like NVDA.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2110/">Bug #2110</a>,
+	<a href="https://sourceforge.net/p/scintilla/bugs/2114/">Bug #2114</a>.
+	</li>
+ 	<li>
+	Fix deletion of isolated invalid bytes.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2116/">Bug #2116</a>.
+	</li>
+ 	<li>
+	Fix position of line caret when overstrike caret set to block.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2106/">Bug #2106</a>.
+	</li>
+    </ul>
+    <h3>
+       <a href="https://www.scintilla.org/scite417.zip">Release 4.1.7</a>
+    </h3>
+    <ul>
+	<li>
+	Released 13 June 2019.
+	</li>
+	<li>
+	Fixes an incorrect default setting in SciTE which caused multiple visual features to fail to display.
+	</li>
+    </ul>
+    <h3>
+       <a href="https://www.scintilla.org/scite416.zip">Release 4.1.6</a>
+    </h3>
+    <ul>
+	<li>
+	Released 10 June 2019.
+	</li>
+	<li>
+	For Visual C++ 2019, /std:c++latest now includes some C++20 features so switch to /std:c++17.
+	</li>
+	<li>
+	SciTE supports editing files larger than 2 gigabytes when built as a 64-bit application.
+	</li>
+ 	<li>
+	Lexer added for X12.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1280/">Feature #1280</a>.
+	</li>
+ 	<li>
+	CMake folder folds function - endfunction.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1289/">Feature #1289</a>.
+	</li>
+ 	<li>
+	VB lexer adds support for VB2017 binary literal &amp;B and digit separators 123_456.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1288/">Feature #1288</a>.
+	</li>
+	<li>
+	Improved performance of line folding code on large files when no folds are contracted.
+	This improves the time taken to open or close large files.
+	</li>
+	<li>
+	Fix bug where changing identifier sets in lexers preserved previous identifiers.
+	</li>
+	<li>
+	Fixed bug where changing to Unicode would rediscover line end positions even if still
+	sticking to ASCII (not Unicode NEL, LS, PS) line ends.
+	Only noticeable on huge files with over 100,000 lines.
+	</li>
+ 	<li>
+	Changed behaviour of SCI_STYLESETCASE(*,SC_CASE_CAMEL) so that it only treats 'a-zA-Z'
+	as word characters because this covers the feature's intended use (viewing case-insensitive ASCII-only
+	keywords in a specified casing style) and simplifies the behaviour and code.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1238/">Feature #1238</a>.
+	</li>
+ 	<li>
+	In SciTE added Camel case option "case:c" for styles to show keywords with initial capital.
+	</li>
+    </ul>
+    <h3>
+       <a href="https://www.scintilla.org/scite415.zip">Release 4.1.5</a>
+    </h3>
+    <ul>
+	<li>
+	Released 17 April 2019.
+	</li>
+ 	<li>
+	On Win32, removed special handling of non-0 wParam to WM_PAINT.
+	</li>
+ 	<li>
+	Implement high-priority idle on Win32 to make redraw smoother and more efficient.
+	</li>
+	<li>
+	Add vertical bookmark symbol SC_MARK_VERTICALBOOKMARK.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1276/">Feature #1276</a>.
+	</li>
+ 	<li>
+	Set default fold display text SCI_SETDEFAULTFOLDDISPLAYTEXT(text).
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1272/">Feature #1272</a>.
+	</li>
+ 	<li>
+	Add SCI_SETCHARACTERCATEGORYOPTIMIZATION API to optimize speed
+	of character category features like determining whether a character is a space or number
+	at the expense of memory.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1259/">Feature #1259</a>.
+	</li>
+ 	<li>
+	Improve the styling of numbers in Nim.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1268/">Feature #1268</a>.
+	</li>
+ 	<li>
+	Fix exception when inserting DBCS text.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2093/">Bug #2093</a>.
+	</li>
+	<li>
+	Improve performance of accessibility on GTK.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2094/">Bug #2094</a>.
+	</li>
+ 	<li>
+	Fix text reported for deletion with accessibility on GTK.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2095/">Bug #2095</a>.
+	</li>
+ 	<li>
+	Fix flicker when inserting primary selection on GTK.
+	<a href="https://sourceforge.net/p/scintilla/bugs/2087/">Bug #2087</a>.
+	</li>
+ 	<li>
+	Support coloured text in Windows 8.1+.
+	<a href="https://sourceforge.net/p/scintilla/feature-requests/1277/">Feature #1277</a>.
+	</li>
+ 	<li>
+	Avoid potential long hangs with idle styling for huge documents on Cocoa and GTK.
+	</li>
+    </ul>
     <h3>
        <a href="https://www.scintilla.org/scite414.zip">Release 4.1.4</a>
     </h3>
@@ -1217,7 +1388,7 @@
 	</li>
 	<li>
 	This is the first release of the
-	<a href="https://www.scintilla.org/LongTermDownload.html">long term branch</a>
+	<a href="https://scintilla.sourceforge.io/LongTermDownload.html">long term branch</a>
 	which avoids using features from C++14 or later in order to support older systems.
 	</li>
 	<li>

scintilla/doc/index.html

@@ -9,7 +9,7 @@
     <meta name="keywords" content="Scintilla, SciTE, Editing Component, Text Editor" />
     <meta name="Description"
     content="www.scintilla.org is the home of the Scintilla editing component and SciTE text editor application." />
-    <meta name="Date.Modified" content="20190307" />
+    <meta name="Date.Modified" content="20190705" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <style type="text/css">
         #versionlist {
@@ -53,11 +53,11 @@
         </td>
         <td width="40%" align="left">
           <font color="#FFCC99" size="4"> A free source code editing component for Win32,
-          GTK+, and OS X</font>
+          GTK, and OS X</font>
         </td>
         <td width="40%" align="right">
-          <font color="#FFCC99" size="3"> Release version 4.1.4<br />
+          <font color="#FFCC99" size="3"> Release version 4.2.0<br />
-           Site last modified March 7 2019</font>
+           Site last modified July 5 2019</font>
         </td>
         <td width="20%">
           &nbsp;
@@ -72,11 +72,12 @@
       </tr>
     </table>
     <ul id="versionlist">
+      <li>Version 4.2.0 adds new types to Scintilla.iface.</li>
+      <li>Version 4.1.7 is just to match a bug-fix release of SciTE and changes nothing in Scintilla.</li>
+      <li>Version 4.1.6 adds an X12 lexer and improves performance on huge files.</li>
+      <li>Version 4.1.5 removes special handling of non-0 wParam to WM_PAINT on Win32 and changes
+      clipboard operations on GTK to avoid flickering.</li>
       <li>Version 4.1.4 makes calltips work on Qt and adds a lexer for .NET's Common Intermediate Language CIL.</li>
-      <li>Version 4.1.3 is compatible with macOS 10.14, improves performance, and adds a new lexer for Nim which will replace the "nimrod" lexer.</li>
-      <li>Version 4.1.2 is a minor release.</li>
-      <li>Version 4.1.1 adds optional indexing of line starts in UTF-8 documents by UTF-32 code points and UTF-16 code units.</li>
-      <li>Version 4.1.0 adds experimental support for bidirectional text as used by Arabic and Hebrew on Win32 and Cocoa.</li>
     </ul>
     <ul id="menu">
       <li id="remote1"><a href="https://www.scintilla.org/SciTEImage.html">Screenshot</a></li>
@@ -133,8 +134,8 @@ if (!IsRemote()) { //if NOT remote...
     </p>
     <p>
        Scintilla and SciTE are currently available for Intel Win32, OS X, and Linux compatible operating
-      systems with GTK+. They have been run on Windows XP, Windows 7, OS X 10.7+, and on Ubuntu 14.04
+      systems with GTK. They have been run on Windows XP, Windows 7, OS X 10.7+, and on Ubuntu 14.04
-      with GTK+ 2.24. <a href="https://www.scintilla.org/SciTEImage.html">Here is a screenshot of
+      with GTK 2.24. <a href="https://www.scintilla.org/SciTEImage.html">Here is a screenshot of
       SciTE.</a><br />
     </p>
     <p>

scintilla/gtk/Converter.h

@@ -15,14 +15,14 @@ const gsize sizeFailure = static_cast<gsize>(-1);
  */
 class Converter {
 	GIConv iconvh;
-	void OpenHandle(const char *fullDestination, const char *charSetSource) {
+	void OpenHandle(const char *fullDestination, const char *charSetSource) noexcept {
 		iconvh = g_iconv_open(fullDestination, charSetSource);
 	}
-	bool Succeeded() const {
+	bool Succeeded() const noexcept {
 		return iconvh != iconvhBad;
 	}
 public:
-	Converter() {
+	Converter() noexcept {
 		iconvh = iconvhBad;
 	}
 	Converter(const char *charSetDestination, const char *charSetSource, bool transliterations) {
@@ -32,7 +32,7 @@ public:
 	~Converter() {
 		Close();
 	}
-	operator bool() const {
+	operator bool() const noexcept {
 		return Succeeded();
 	}
 	void Open(const char *charSetDestination, const char *charSetSource, bool transliterations) {
@@ -50,13 +50,13 @@ public:
 			}
 		}
 	}
-	void Close() {
+	void Close() noexcept {
 		if (Succeeded()) {
 			g_iconv_close(iconvh);
 			iconvh = iconvhBad;
 		}
 	}
-	gsize Convert(char** src, gsize *srcleft, char **dst, gsize *dstleft) const {
+	gsize Convert(char **src, gsize *srcleft, char **dst, gsize *dstleft) const noexcept {
 		if (!Succeeded()) {
 			return sizeFailure;
 		} else {

scintilla/gtk/DepGen.py

@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# DepGen.py - produce a make dependencies file for Scintilla
+# Copyright 2019 by Neil Hodgson <neilh@scintilla.org>
+# The License.txt file describes the conditions under which this software may be distributed.
+# Requires Python 2.7 or later
+
+import sys
+
+sys.path.append("..")
+
+from scripts import Dependencies
+
+topComment = "# Created by DepGen.py. To recreate, run 'python DepGen.py'.\n"
+
+def Generate():
+	sources = ["../src/*.cxx", "../lexlib/*.cxx", "../lexers/*.cxx"]
+	includes = ["../include", "../src", "../lexlib"]
+
+	deps = Dependencies.FindDependencies(["../gtk/*.cxx"] + sources, ["../gtk"] + includes, ".o", "../gtk/")
+	Dependencies.UpdateDependencies("../gtk/deps.mak", deps, topComment)
+
+if __name__ == "__main__":
+	Generate()

scintilla/gtk/ScintillaGTK.h

@@ -36,7 +36,6 @@ class ScintillaGTK : public ScintillaBase {
 
 	GtkWidgetClass *parentClass;
 
-	static GdkAtom atomClipboard;
 	static GdkAtom atomUTF8;
 	static GdkAtom atomString;
 	static GdkAtom atomUriList;
@@ -80,7 +79,7 @@ public:
 	ScintillaGTK &operator=(ScintillaGTK &&) = delete;
 	virtual ~ScintillaGTK();
 	static ScintillaGTK *FromWidget(GtkWidget *widget);
-	static void ClassInit(OBJECT_CLASS* object_class, GtkWidgetClass *widget_class, GtkContainerClass *container_class);
+	static void ClassInit(OBJECT_CLASS *object_class, GtkWidgetClass *widget_class, GtkContainerClass *container_class);
 private:
 	void Init();
 	void Finalise() override;
@@ -99,7 +98,7 @@ private:
 		TickReason reason;
 		ScintillaGTK *scintilla;
 		guint timer;
-		TimeThunk() : reason(tickCaret), scintilla(NULL), timer(0) {}
+		TimeThunk() noexcept : reason(tickCaret), scintilla(nullptr), timer(0) {}
 	};
 	TimeThunk timers[tickDwell+1];
 	bool FineTickerRunning(TickReason reason) override;
@@ -127,18 +126,24 @@ private:
 	int KeyDefault(int key, int modifiers) override;
 	void CopyToClipboard(const SelectionText &selectedText) override;
 	void Copy() override;
+	void RequestSelection(GdkAtom atomSelection);
 	void Paste() override;
 	void CreateCallTipWindow(PRectangle rc) override;
 	void AddToPopUp(const char *label, int cmd = 0, bool enabled = true) override;
 	bool OwnPrimarySelection();
 	void ClaimSelection() override;
 	void GetGtkSelectionText(GtkSelectionData *selectionData, SelectionText &selText);
+	void InsertSelection(GtkSelectionData *selectionData);
+public:	// Public for SelectionReceiver
+	GObject *MainObject() const noexcept;
+	void ReceivedClipboard(GtkSelectionData *selection_data) noexcept;
+private:
 	void ReceivedSelection(GtkSelectionData *selection_data);
 	void ReceivedDrop(GtkSelectionData *selection_data);
 	static void GetSelection(GtkSelectionData *selection_data, guint info, SelectionText *text);
 	void StoreOnClipboard(SelectionText *clipText);
-	static void ClipboardGetSelection(GtkClipboard* clip, GtkSelectionData *selection_data, guint info, void *data);
+	static void ClipboardGetSelection(GtkClipboard *clip, GtkSelectionData *selection_data, guint info, void *data);
-	static void ClipboardClearSelection(GtkClipboard* clip, void *data);
+	static void ClipboardClearSelection(GtkClipboard *clip, void *data);
 
 	void UnclaimSelection(GdkEventSelection *selection_event);
 	static void PrimarySelection(GtkWidget *widget, GtkSelectionData *selection_data, guint info, guint time_stamp, ScintillaGTK *sciThis);
@@ -195,8 +200,8 @@ private:
 	gboolean ExposePreeditThis(GtkWidget *widget, GdkEventExpose *ose);
 	static gboolean ExposePreedit(GtkWidget *widget, GdkEventExpose *ose, ScintillaGTK *sciThis);
 #endif
-	AtkObject* GetAccessibleThis(GtkWidget *widget);
+	AtkObject *GetAccessibleThis(GtkWidget *widget);
-	static AtkObject* GetAccessible(GtkWidget *widget);
+	static AtkObject *GetAccessible(GtkWidget *widget);
 
 	bool KoreanIME();
 	void CommitThis(char *commitStr);
@@ -208,8 +213,8 @@ private:
 	void DrawImeIndicator(int indicator, int len);
 	void SetCandidateWindowPos();
 
-	static void StyleSetText(GtkWidget *widget, GtkStyle *previous, void*);
+	static void StyleSetText(GtkWidget *widget, GtkStyle *previous, void *);
-	static void RealizeText(GtkWidget *widget, void*);
+	static void RealizeText(GtkWidget *widget, void *);
 	static void Dispose(GObject *object);
 	static void Destroy(GObject *object);
 	static void SelectionReceived(GtkWidget *widget, GtkSelectionData *selection_data,
@@ -256,11 +261,11 @@ class GObjectWatcher {
 		PLATFORM_ASSERT(obj == weakRef);
 
 		Destroyed();
-		weakRef = 0;
+		weakRef = nullptr;
 	}
 
 	static void WeakNotify(gpointer data, GObject *obj) {
-		static_cast<GObjectWatcher*>(data)->WeakNotifyThis(obj);
+		static_cast<GObjectWatcher *>(data)->WeakNotifyThis(obj);
 	}
 
 public:
@@ -278,7 +283,7 @@ public:
 	virtual void Destroyed() {}
 
 	bool IsDestroyed() const {
-		return weakRef != 0;
+		return weakRef != nullptr;
 	}
 };
 

scintilla/gtk/ScintillaGTKAccessible.cxx

@@ -78,7 +78,10 @@
 # include <gtk/gtk-a11y.h>
 #endif
 
-#if defined(__WIN32__) || defined(_MSC_VER)
+#if defined(_WIN32)
+// On Win32 use windows.h to access CLIPFORMAT
+#undef NOMINMAX
+#define NOMINMAX
 #include <windows.h>
 #endif
 
@@ -93,6 +96,7 @@
 #include "SciLexer.h"
 #endif
 #include "StringCopy.h"
+#include "CharacterCategory.h"
 #ifdef SCI_LEXER
 #include "LexerModule.h"
 #endif
@@ -148,7 +152,7 @@ ScintillaGTKAccessible *ScintillaGTKAccessible::FromAccessible(GtkAccessible *ac
 	// FIXME: do we need the check below?  GTK checks that in all methods, so maybe
 	GtkWidget *widget = gtk_accessible_get_widget(accessible);
 	if (! widget) {
-		return 0;
+		return nullptr;
 	}
 
 	return SCINTILLA_OBJECT_ACCESSIBLE_GET_PRIVATE(accessible)->pscin;
@@ -157,23 +161,23 @@ ScintillaGTKAccessible *ScintillaGTKAccessible::FromAccessible(GtkAccessible *ac
 ScintillaGTKAccessible::ScintillaGTKAccessible(GtkAccessible *accessible_, GtkWidget *widget_) :
 		accessible(accessible_),
 		sci(ScintillaGTK::FromWidget(widget_)),
-		deletionLengthChar(0),
 		old_pos(-1) {
+	SetAccessibility(true);
 	g_signal_connect(widget_, "sci-notify", G_CALLBACK(SciNotify), this);
 }
 
 ScintillaGTKAccessible::~ScintillaGTKAccessible() {
 	if (gtk_accessible_get_widget(accessible)) {
-		g_signal_handlers_disconnect_matched(sci->sci, G_SIGNAL_MATCH_DATA, 0, 0, NULL, NULL, this);
+		g_signal_handlers_disconnect_matched(sci->sci, G_SIGNAL_MATCH_DATA, 0, 0, nullptr, nullptr, this);
 	}
 }
 
 gchar *ScintillaGTKAccessible::GetTextRangeUTF8(Sci::Position startByte, Sci::Position endByte) {
-	g_return_val_if_fail(startByte >= 0, NULL);
+	g_return_val_if_fail(startByte >= 0, nullptr);
 	// FIXME: should we swap start/end if necessary?
-	g_return_val_if_fail(endByte >= startByte, NULL);
+	g_return_val_if_fail(endByte >= startByte, nullptr);
 
-	gchar *utf8Text = NULL;
+	gchar *utf8Text = nullptr;
 	const char *charSetBuffer;
 
 	// like TargetAsUTF8, but avoids a double conversion
@@ -208,7 +212,7 @@ gchar *ScintillaGTKAccessible::GetText(int startChar, int endChar) {
 
 gchar *ScintillaGTKAccessible::GetTextAfterOffset(int charOffset,
 		AtkTextBoundary boundaryType, int *startChar, int *endChar) {
-	g_return_val_if_fail(charOffset >= 0, NULL);
+	g_return_val_if_fail(charOffset >= 0, nullptr);
 
 	Sci::Position startByte, endByte;
 	Sci::Position byteOffset = ByteOffsetFromCharacterOffset(charOffset);
@@ -250,7 +254,7 @@ gchar *ScintillaGTKAccessible::GetTextAfterOffset(int charOffset,
 
 		default:
 			*startChar = *endChar = -1;
-			return NULL;
+			return nullptr;
 	}
 
 	CharacterRangeFromByteRange(startByte, endByte, startChar, endChar);
@@ -259,7 +263,7 @@ gchar *ScintillaGTKAccessible::GetTextAfterOffset(int charOffset,
 
 gchar *ScintillaGTKAccessible::GetTextBeforeOffset(int charOffset,
 		AtkTextBoundary boundaryType, int *startChar, int *endChar) {
-	g_return_val_if_fail(charOffset >= 0, NULL);
+	g_return_val_if_fail(charOffset >= 0, nullptr);
 
 	Sci::Position startByte, endByte;
 	Sci::Position byteOffset = ByteOffsetFromCharacterOffset(charOffset);
@@ -312,7 +316,7 @@ gchar *ScintillaGTKAccessible::GetTextBeforeOffset(int charOffset,
 
 		default:
 			*startChar = *endChar = -1;
-			return NULL;
+			return nullptr;
 	}
 
 	CharacterRangeFromByteRange(startByte, endByte, startChar, endChar);
@@ -321,7 +325,7 @@ gchar *ScintillaGTKAccessible::GetTextBeforeOffset(int charOffset,
 
 gchar *ScintillaGTKAccessible::GetTextAtOffset(int charOffset,
 		AtkTextBoundary boundaryType, int *startChar, int *endChar) {
-	g_return_val_if_fail(charOffset >= 0, NULL);
+	g_return_val_if_fail(charOffset >= 0, nullptr);
 
 	Sci::Position startByte, endByte;
 	Sci::Position byteOffset = ByteOffsetFromCharacterOffset(charOffset);
@@ -374,7 +378,7 @@ gchar *ScintillaGTKAccessible::GetTextAtOffset(int charOffset,
 
 		default:
 			*startChar = *endChar = -1;
-			return NULL;
+			return nullptr;
 	}
 
 	CharacterRangeFromByteRange(startByte, endByte, startChar, endChar);
@@ -384,7 +388,7 @@ gchar *ScintillaGTKAccessible::GetTextAtOffset(int charOffset,
 #if ATK_CHECK_VERSION(2, 10, 0)
 gchar *ScintillaGTKAccessible::GetStringAtOffset(int charOffset,
 		AtkTextGranularity granularity, int *startChar, int *endChar) {
-	g_return_val_if_fail(charOffset >= 0, NULL);
+	g_return_val_if_fail(charOffset >= 0, nullptr);
 
 	Sci::Position startByte, endByte;
 	Sci::Position byteOffset = ByteOffsetFromCharacterOffset(charOffset);
@@ -406,7 +410,7 @@ gchar *ScintillaGTKAccessible::GetStringAtOffset(int charOffset,
 		}
 		default:
 			*startChar = *endChar = -1;
-			return NULL;
+			return nullptr;
 	}
 
 	CharacterRangeFromByteRange(startByte, endByte, startChar, endChar);
@@ -528,10 +532,10 @@ static AtkAttributeSet *AddTextColorAttribute(AtkAttributeSet *attributes, AtkTe
 }
 
 AtkAttributeSet *ScintillaGTKAccessible::GetAttributesForStyle(unsigned int styleNum) {
-	AtkAttributeSet *attr_set = NULL;
+	AtkAttributeSet *attr_set = nullptr;
 
 	if (styleNum >= sci->vs.styles.size())
-		return NULL;
+		return nullptr;
 	Style &style = sci->vs.styles[styleNum];
 
 	attr_set = AddTextAttribute(attr_set, ATK_TEXT_ATTR_FAMILY_NAME, g_strdup(style.fontName));
@@ -548,7 +552,7 @@ AtkAttributeSet *ScintillaGTKAccessible::GetAttributesForStyle(unsigned int styl
 }
 
 AtkAttributeSet *ScintillaGTKAccessible::GetRunAttributes(int charOffset, int *startChar, int *endChar) {
-	g_return_val_if_fail(charOffset >= -1, NULL);
+	g_return_val_if_fail(charOffset >= -1, nullptr);
 
 	Sci::Position byteOffset;
 	if (charOffset == -1) {
@@ -558,7 +562,7 @@ AtkAttributeSet *ScintillaGTKAccessible::GetRunAttributes(int charOffset, int *s
 	}
 	int length = sci->pdoc->Length();
 
-	g_return_val_if_fail(byteOffset <= length, NULL);
+	g_return_val_if_fail(byteOffset <= length, nullptr);
 
 	const char style = StyleAt(byteOffset, true);
 	// compute the range for this style
@@ -584,7 +588,7 @@ gint ScintillaGTKAccessible::GetNSelections() {
 
 gchar *ScintillaGTKAccessible::GetSelection(gint selection_num, int *startChar, int *endChar) {
 	if (selection_num < 0 || (unsigned int) selection_num >= sci->sel.Count())
-		return NULL;
+		return nullptr;
 
 	Sci::Position startByte = sci->sel.Range(selection_num).Start().Position();
 	Sci::Position endByte = sci->sel.Range(selection_num).End().Position();
@@ -738,7 +742,7 @@ void ScintillaGTKAccessible::PasteText(int charPosition) {
 		Sci::Position bytePosition;
 
 		void Destroyed() override {
-			scia = 0;
+			scia = nullptr;
 		}
 
 		Helper(ScintillaGTKAccessible *scia_, Sci::Position bytePos_) :
@@ -764,7 +768,7 @@ void ScintillaGTKAccessible::PasteText(int charPosition) {
 		static void TextReceivedCallback(GtkClipboard *clipboard, const gchar *text, gpointer data) {
 			Helper *helper = static_cast<Helper*>(data);
 			try {
-				if (helper->scia != 0) {
+				if (helper->scia != nullptr) {
 					helper->TextReceived(clipboard, text);
 				}
 			} catch (...) {}
@@ -862,10 +866,12 @@ void ScintillaGTKAccessible::NotifyReadOnly() {
 #endif
 }
 
-void ScintillaGTKAccessible::SetAccessibility() {
+void ScintillaGTKAccessible::SetAccessibility(bool enabled) {
 	// Called by ScintillaGTK when application has enabled or disabled accessibility
-	character_offsets.resize(0);
+	if (enabled)
-	character_offsets.push_back(0);
+		sci->pdoc->AllocateLineCharacterIndex(SC_LINECHARACTERINDEX_UTF32);
+	else
+		sci->pdoc->ReleaseLineCharacterIndex(SC_LINECHARACTERINDEX_UTF32);
 }
 
 void ScintillaGTKAccessible::Notify(GtkWidget *, gint, SCNotification *nt) {
@@ -873,13 +879,6 @@ void ScintillaGTKAccessible::Notify(GtkWidget *, gint, SCNotification *nt) {
 		return;
 	switch (nt->nmhdr.code) {
 		case SCN_MODIFIED: {
-			if (nt->modificationType & (SC_MOD_INSERTTEXT | SC_MOD_DELETETEXT)) {
-				// invalidate character offset cache if applicable
-				const Sci::Line line = sci->pdoc->LineFromPosition(nt->position);
-				if (character_offsets.size() > static_cast<size_t>(line + 1)) {
-					character_offsets.resize(line + 1);
-				}
-			}
 			if (nt->modificationType & SC_MOD_INSERTTEXT) {
 				int startChar = CharacterOffsetFromByteOffset(nt->position);
 				int lengthChar = sci->pdoc->CountCharacters(nt->position, nt->position + nt->length);
@@ -887,14 +886,11 @@ void ScintillaGTKAccessible::Notify(GtkWidget *, gint, SCNotification *nt) {
 				UpdateCursor();
 			}
 			if (nt->modificationType & SC_MOD_BEFOREDELETE) {
-				// We cannot compute the deletion length in DELETETEXT as it requires accessing the
+				int startChar = CharacterOffsetFromByteOffset(nt->position);
-				// buffer, so that the character are still present.  So, we cache the value here,
+				int lengthChar = sci->pdoc->CountCharacters(nt->position, nt->position + nt->length);
-				// and use it in DELETETEXT that fires quickly after.
+				g_signal_emit_by_name(accessible, "text-changed::delete", startChar, lengthChar);
-				deletionLengthChar = sci->pdoc->CountCharacters(nt->position, nt->position + nt->length);
 			}
 			if (nt->modificationType & SC_MOD_DELETETEXT) {
-				int startChar = CharacterOffsetFromByteOffset(nt->position);
-				g_signal_emit_by_name(accessible, "text-changed::delete", startChar, deletionLengthChar);
 				UpdateCursor();
 			}
 			if (nt->modificationType & SC_MOD_CHANGESTYLE) {
@@ -926,20 +922,20 @@ void ScintillaGTKAccessible::Notify(GtkWidget *, gint, SCNotification *nt) {
 
 // AtkText
 gchar *ScintillaGTKAccessible::AtkTextIface::GetText(AtkText *text, int start_offset, int end_offset) {
-	WRAPPER_METHOD_BODY(text, GetText(start_offset, end_offset), NULL);
+	WRAPPER_METHOD_BODY(text, GetText(start_offset, end_offset), nullptr);
 }
 gchar *ScintillaGTKAccessible::AtkTextIface::GetTextAfterOffset(AtkText *text, int offset, AtkTextBoundary boundary_type, int *start_offset, int *end_offset) {
-	WRAPPER_METHOD_BODY(text, GetTextAfterOffset(offset, boundary_type, start_offset, end_offset), NULL)
+	WRAPPER_METHOD_BODY(text, GetTextAfterOffset(offset, boundary_type, start_offset, end_offset), nullptr)
 }
 gchar *ScintillaGTKAccessible::AtkTextIface::GetTextBeforeOffset(AtkText *text, int offset, AtkTextBoundary boundary_type, int *start_offset, int *end_offset) {
-	WRAPPER_METHOD_BODY(text, GetTextBeforeOffset(offset, boundary_type, start_offset, end_offset), NULL)
+	WRAPPER_METHOD_BODY(text, GetTextBeforeOffset(offset, boundary_type, start_offset, end_offset), nullptr)
 }
 gchar *ScintillaGTKAccessible::AtkTextIface::GetTextAtOffset(AtkText *text, gint offset, AtkTextBoundary boundary_type, gint *start_offset, gint *end_offset) {
-	WRAPPER_METHOD_BODY(text, GetTextAtOffset(offset, boundary_type, start_offset, end_offset), NULL)
+	WRAPPER_METHOD_BODY(text, GetTextAtOffset(offset, boundary_type, start_offset, end_offset), nullptr)
 }
 #if ATK_CHECK_VERSION(2, 10, 0)
 gchar *ScintillaGTKAccessible::AtkTextIface::GetStringAtOffset(AtkText *text, gint offset, AtkTextGranularity granularity, gint *start_offset, gint *end_offset) {
-	WRAPPER_METHOD_BODY(text, GetStringAtOffset(offset, granularity, start_offset, end_offset), NULL)
+	WRAPPER_METHOD_BODY(text, GetStringAtOffset(offset, granularity, start_offset, end_offset), nullptr)
 }
 #endif
 gunichar ScintillaGTKAccessible::AtkTextIface::GetCharacterAtOffset(AtkText *text, gint offset) {
@@ -961,16 +957,16 @@ void ScintillaGTKAccessible::AtkTextIface::GetCharacterExtents(AtkText *text, gi
 	WRAPPER_METHOD_BODY(text, GetCharacterExtents(offset, x, y, width, height, coords), )
 }
 AtkAttributeSet *ScintillaGTKAccessible::AtkTextIface::GetRunAttributes(AtkText *text, gint offset, gint *start_offset, gint *end_offset) {
-	WRAPPER_METHOD_BODY(text, GetRunAttributes(offset, start_offset, end_offset), NULL)
+	WRAPPER_METHOD_BODY(text, GetRunAttributes(offset, start_offset, end_offset), nullptr)
 }
 AtkAttributeSet *ScintillaGTKAccessible::AtkTextIface::GetDefaultAttributes(AtkText *text) {
-	WRAPPER_METHOD_BODY(text, GetDefaultAttributes(), NULL)
+	WRAPPER_METHOD_BODY(text, GetDefaultAttributes(), nullptr)
 }
 gint ScintillaGTKAccessible::AtkTextIface::GetNSelections(AtkText *text) {
 	WRAPPER_METHOD_BODY(text, GetNSelections(), 0)
 }
 gchar *ScintillaGTKAccessible::AtkTextIface::GetSelection(AtkText *text, gint selection_num, gint *start_pos, gint *end_pos) {
-	WRAPPER_METHOD_BODY(text, GetSelection(selection_num, start_pos, end_pos), NULL)
+	WRAPPER_METHOD_BODY(text, GetSelection(selection_num, start_pos, end_pos), nullptr)
 }
 gboolean ScintillaGTKAccessible::AtkTextIface::AddSelection(AtkText *text, gint start, gint end) {
 	WRAPPER_METHOD_BODY(text, AddSelection(start, end), FALSE)
@@ -1009,7 +1005,7 @@ static GType scintilla_object_accessible_factory_get_type(void);
 
 static void scintilla_object_accessible_init(ScintillaObjectAccessible *accessible);
 static void scintilla_object_accessible_class_init(ScintillaObjectAccessibleClass *klass);
-static gpointer scintilla_object_accessible_parent_class = NULL;
+static gpointer scintilla_object_accessible_parent_class = nullptr;
 
 
 // @p parent_type is only required on GTK 3.2 to 3.6, and only on the first call
@@ -1019,27 +1015,27 @@ static GType scintilla_object_accessible_get_type(GType parent_type G_GNUC_UNUSE
 	if (g_once_init_enter(&type_id_result)) {
 		GTypeInfo tinfo = {
 			0,															/* class size */
-			(GBaseInitFunc) NULL,										/* base init */
+			(GBaseInitFunc) nullptr,										/* base init */
-			(GBaseFinalizeFunc) NULL,									/* base finalize */
+			(GBaseFinalizeFunc) nullptr,									/* base finalize */
 			(GClassInitFunc) scintilla_object_accessible_class_init,	/* class init */
-			(GClassFinalizeFunc) NULL,									/* class finalize */
+			(GClassFinalizeFunc) nullptr,									/* class finalize */
-			NULL,														/* class data */
+			nullptr,														/* class data */
 			0,															/* instance size */
 			0,															/* nb preallocs */
 			(GInstanceInitFunc) scintilla_object_accessible_init,		/* instance init */
-			NULL														/* value table */
+			nullptr														/* value table */
 		};
 
 		const GInterfaceInfo atk_text_info = {
 			(GInterfaceInitFunc) ScintillaGTKAccessible::AtkTextIface::init,
-			(GInterfaceFinalizeFunc) NULL,
+			(GInterfaceFinalizeFunc) nullptr,
-			NULL
+			nullptr
 		};
 
 		const GInterfaceInfo atk_editable_text_info = {
 			(GInterfaceInitFunc) ScintillaGTKAccessible::AtkEditableTextIface::init,
-			(GInterfaceFinalizeFunc) NULL,
+			(GInterfaceFinalizeFunc) nullptr,
-			NULL
+			nullptr
 		};
 
 #if HAVE_GTK_A11Y_H
@@ -1080,13 +1076,13 @@ static GType scintilla_object_accessible_get_type(GType parent_type G_GNUC_UNUSE
 }
 
 static AtkObject *scintilla_object_accessible_new(GType parent_type, GObject *obj) {
-	g_return_val_if_fail(SCINTILLA_IS_OBJECT(obj), NULL);
+	g_return_val_if_fail(SCINTILLA_IS_OBJECT(obj), nullptr);
 
 	AtkObject *accessible = (AtkObject *) g_object_new(scintilla_object_accessible_get_type(parent_type),
 #if HAVE_WIDGET_SET_UNSET
 		"widget", obj,
 #endif
-		NULL);
+		nullptr);
 	atk_object_initialize(accessible, obj);
 
 	return accessible;
@@ -1098,7 +1094,7 @@ static AtkObject *scintilla_object_accessible_new(GType parent_type, GObject *ob
 // @p cache pointer to store the AtkObject between repeated calls.  Might or might not be filled.
 // @p widget_parent_class pointer to the widget's parent class (to chain up method calls).
 AtkObject *ScintillaGTKAccessible::WidgetGetAccessibleImpl(GtkWidget *widget, AtkObject **cache, gpointer widget_parent_class G_GNUC_UNUSED) {
-	if (*cache != NULL) {
+	if (*cache != nullptr) {
 		return *cache;
 	}
 
@@ -1148,7 +1144,7 @@ static AtkStateSet *scintilla_object_accessible_ref_state_set(AtkObject *accessi
 	AtkStateSet *state_set = ATK_OBJECT_CLASS(scintilla_object_accessible_parent_class)->ref_state_set(accessible);
 
 	GtkWidget *widget = gtk_accessible_get_widget(GTK_ACCESSIBLE(accessible));
-	if (widget == NULL) {
+	if (widget == nullptr) {
 		atk_state_set_add_state(state_set, ATK_STATE_DEFUNCT);
 	} else {
 		if (! scintilla_send_message(SCINTILLA_OBJECT(widget), SCI_GETREADONLY, 0, 0))
@@ -1168,11 +1164,11 @@ static AtkStateSet *scintilla_object_accessible_ref_state_set(AtkObject *accessi
 
 static void scintilla_object_accessible_widget_set(GtkAccessible *accessible) {
 	GtkWidget *widget = gtk_accessible_get_widget(accessible);
-	if (widget == NULL)
+	if (widget == nullptr)
 		return;
 
 	ScintillaObjectAccessiblePrivate *priv = SCINTILLA_OBJECT_ACCESSIBLE_GET_PRIVATE(accessible);
-	if (priv->pscin != 0)
+	if (priv->pscin)
 		delete priv->pscin;
 	priv->pscin = new ScintillaGTKAccessible(accessible, widget);
 }
@@ -1180,7 +1176,7 @@ static void scintilla_object_accessible_widget_set(GtkAccessible *accessible) {
 #if HAVE_WIDGET_SET_UNSET
 static void scintilla_object_accessible_widget_unset(GtkAccessible *accessible) {
 	GtkWidget *widget = gtk_accessible_get_widget(accessible);
-	if (widget == NULL)
+	if (widget == nullptr)
 		return;
 
 	ScintillaObjectAccessiblePrivate *priv = SCINTILLA_OBJECT_ACCESSIBLE_GET_PRIVATE(accessible);
@@ -1204,7 +1200,7 @@ static void scintilla_object_accessible_finalize(GObject *object) {
 
 	if (priv->pscin) {
 		delete priv->pscin;
-		priv->pscin = 0;
+		priv->pscin = nullptr;
 	}
 
 	G_OBJECT_CLASS(scintilla_object_accessible_parent_class)->finalize(object);
@@ -1233,7 +1229,7 @@ static void scintilla_object_accessible_class_init(ScintillaObjectAccessibleClas
 static void scintilla_object_accessible_init(ScintillaObjectAccessible *accessible) {
 	ScintillaObjectAccessiblePrivate *priv = SCINTILLA_OBJECT_ACCESSIBLE_GET_PRIVATE(accessible);
 
-	priv->pscin = 0;
+	priv->pscin = nullptr;
 }
 
 #if HAVE_GTK_FACTORY

scintilla/gtk/ScintillaGTKAccessible.h

@@ -18,11 +18,6 @@ private:
 	GtkAccessible *accessible;
 	ScintillaGTK *sci;
 
-	// cache holding character offset for each line start, see CharacterOffsetFromByteOffset()
-	std::vector<Sci::Position> character_offsets;
-
-	// cached length of the deletion, in characters (see Notify())
-	int deletionLengthChar;
 	// local state for comparing
 	Sci::Position old_pos;
 	std::vector<SelectionRange> old_sels;
@@ -37,6 +32,19 @@ private:
 	}
 
 	Sci::Position ByteOffsetFromCharacterOffset(Sci::Position startByte, int characterOffset) {
+		if (!(sci->pdoc->LineCharacterIndex() & SC_LINECHARACTERINDEX_UTF32)) {
+			return startByte + characterOffset;
+		}
+		if (characterOffset > 0) {
+			// Try and reduce the range by reverse-looking into the character offset cache
+			Sci::Line lineStart = sci->pdoc->LineFromPosition(startByte);
+			Sci::Position posStart = sci->pdoc->IndexLineStart(lineStart, SC_LINECHARACTERINDEX_UTF32);
+			Sci::Line line = sci->pdoc->LineFromPositionIndex(posStart + characterOffset, SC_LINECHARACTERINDEX_UTF32);
+			if (line != lineStart) {
+				startByte += sci->pdoc->LineStart(line) - sci->pdoc->LineStart(lineStart);
+				characterOffset -= sci->pdoc->IndexLineStart(line, SC_LINECHARACTERINDEX_UTF32) - posStart;
+			}
+		}
 		Sci::Position pos = sci->pdoc->GetRelativePosition(startByte, characterOffset);
 		if (pos == INVALID_POSITION) {
 			// clamp invalid positions inside the document
@@ -54,18 +62,12 @@ private:
 	}
 
 	Sci::Position CharacterOffsetFromByteOffset(Sci::Position byteOffset) {
+		if (!(sci->pdoc->LineCharacterIndex() & SC_LINECHARACTERINDEX_UTF32)) {
+			return byteOffset;
+		}
 		const Sci::Line line = sci->pdoc->LineFromPosition(byteOffset);
-		if (character_offsets.size() <= static_cast<size_t>(line)) {
-			if (character_offsets.empty())
-				character_offsets.push_back(0);
-			for (Sci::Position i = character_offsets.size(); i <= line; i++) {
-				const Sci::Position start = sci->pdoc->LineStart(i - 1);
-				const Sci::Position end = sci->pdoc->LineStart(i);
-				character_offsets.push_back(character_offsets[i - 1] + sci->pdoc->CountCharacters(start, end));
-			}
-		}
 		const Sci::Position lineStart = sci->pdoc->LineStart(line);
-		return character_offsets[line] + sci->pdoc->CountCharacters(lineStart, byteOffset);
+		return sci->pdoc->IndexLineStart(line, SC_LINECHARACTERINDEX_UTF32) + sci->pdoc->CountCharacters(lineStart, byteOffset);
 	}
 
 	void CharacterRangeFromByteRange(Sci::Position startByte, Sci::Position endByte, int *startChar, int *endChar) {
@@ -135,7 +137,7 @@ public:
 	// So ScintillaGTK can notify us
 	void ChangeDocument(Document *oldDoc, Document *newDoc);
 	void NotifyReadOnly();
-	void SetAccessibility();
+	void SetAccessibility(bool enabled);
 
 	// Helper GtkWidget methods
 	static AtkObject *WidgetGetAccessibleImpl(GtkWidget *widget, AtkObject **cache, gpointer widget_parent_class);

scintilla/gtk/makefile

@@ -57,10 +57,7 @@ vpath %.c $(srcdir)
 vpath %.cxx $(srcdir) $(srcdir)/../src $(srcdir)/../lexlib $(srcdir)/../lexers
 
 INCLUDEDIRS=-I $(srcdir)/../include -I $(srcdir)/../src -I $(srcdir)/../lexlib
-ifdef CHECK_DEPRECATED
+CXXBASEFLAGS=$(CXXWARNFLAGS) $(PICFLAGS) -DGTK -DSCI_LEXER $(INCLUDEDIRS)
-DEPRECATED=-DGDK_PIXBUF_DISABLE_DEPRECATED -DGDK_DISABLE_DEPRECATED -DGTK_DISABLE_DEPRECATED -DDISABLE_GDK_FONT
-endif
-CXXBASEFLAGS=$(CXXWARNFLAGS) $(PICFLAGS) -DGTK -DSCI_LEXER $(INCLUDEDIRS) $(DEPRECATED)
 
 ifdef NOTHREADS
 THREADFLAGS=-DG_THREADS_IMPL_NONE
@@ -110,19 +107,15 @@ LEXOBJS:=$(addsuffix .o,$(basename $(sort $(notdir $(wildcard $(srcdir)/../lexer
 analyze:
 	clang --analyze $(CONFIGFLAGS) $(CXXTFLAGS) $(CXXFLAGS) $(srcdir)/*.cxx $(srcdir)/../src/*.cxx $(srcdir)/../lexlib/*.cxx $(srcdir)/../lexers/*.cxx
 
-# To minimize deltas, including between Unix and Windows, the lexer file names are sorted case-insensitively
+depend deps.mak:
-LEXSORTED=$(shell ls $(srcdir)/../lexers/Lex*.cxx | sort -f)
+	python DepGen.py
-DEPSRCS=$(srcdir)/*.cxx $(srcdir)/../src/*.cxx $(srcdir)/../lexlib/*.cxx $(LEXSORTED)
-deps: deps.mak
-deps.mak:
-	$(CXX) -MM $(CONFIGFLAGS) $(CXXTFLAGS) $(DEPSRCS) | sed -e 's/\/usr.* //' | grep [a-zA-Z] > $@
 
 $(COMPLIB): Accessor.o CharacterSet.o DefaultLexer.o LexerBase.o LexerModule.o LexerSimple.o StyleContext.o WordList.o \
 	CharClassify.o Decoration.o Document.o PerLine.o Catalogue.o CallTip.o CaseConvert.o CaseFolder.o \
 	ScintillaBase.o ContractionState.o EditModel.o Editor.o EditView.o ExternalLexer.o MarginView.o \
 	PropSetSimple.o PlatGTK.o \
 	KeyMap.o LineMarker.o PositionCache.o ScintillaGTK.o ScintillaGTKAccessible.o CellBuffer.o CharacterCategory.o ViewStyle.o \
-	RESearch.o RunStyles.o Selection.o Style.o Indicator.o AutoComplete.o UniConversion.o XPM.o \
+	RESearch.o RunStyles.o Selection.o Style.o Indicator.o AutoComplete.o UniConversion.o UniqueString.o XPM.o \
 	$(MARSHALLER) $(LEXOBJS)
 	$(AR) $(ARFLAGS) $@ $^
 	$(RANLIB) $@

scintilla/include/Platform.h

@@ -108,6 +108,18 @@ public:
 		return Point(static_cast<XYPOSITION>(x_), static_cast<XYPOSITION>(y_));
 	}
 
+	bool operator!=(Point other) const noexcept {
+		return (x != other.x) || (y != other.y);
+	}
+
+	Point operator+(Point other) const noexcept {
+		return Point(x + other.x, y + other.y);
+	}
+
+	Point operator-(Point other) const noexcept {
+		return Point(x - other.x, y - other.y);
+	}
+
 	// Other automatically defined methods (assignment, copy constructor, destructor) are fine
 };
 

scintilla/include/SciLexer.h

@@ -140,11 +140,14 @@
 #define SCLEX_SAS 125
 #define SCLEX_NIM 126
 #define SCLEX_CIL 127
+#define SCLEX_X12 128
+#define SCLEX_DATAFLEX 129
 #define SCLEX_SEARCHRESULT 150
 #define SCLEX_OBJC 151
 #define SCLEX_USER 152
 #define SCLEX_AUTOMATIC 1000
 
+
 //For All lexer
 #define SCE_UNIVERSAL_FOUND_STYLE 31
 #define SCE_UNIVERSAL_FOUND_STYLE_SMART 29
@@ -2009,6 +2012,30 @@
 #define SCE_CIL_OPERATOR 8
 #define SCE_CIL_IDENTIFIER 9
 #define SCE_CIL_STRINGEOL 10
+#define SCE_X12_DEFAULT 0
+#define SCE_X12_BAD 1
+#define SCE_X12_ENVELOPE 2
+#define SCE_X12_FUNCTIONGROUP 3
+#define SCE_X12_TRANSACTIONSET 4
+#define SCE_X12_SEGMENTHEADER 5
+#define SCE_X12_SEGMENTEND 6
+#define SCE_X12_SEP_ELEMENT 7
+#define SCE_X12_SEP_SUBELEMENT 8
+#define SCE_DF_DEFAULT 0
+#define SCE_DF_IDENTIFIER 1
+#define SCE_DF_METATAG 2
+#define SCE_DF_IMAGE 3
+#define SCE_DF_COMMENTLINE 4
+#define SCE_DF_PREPROCESSOR 5
+#define SCE_DF_PREPROCESSOR2 6
+#define SCE_DF_NUMBER 7
+#define SCE_DF_HEXNUMBER 8
+#define SCE_DF_WORD 9
+#define SCE_DF_STRING 10
+#define SCE_DF_STRINGEOL 11
+#define SCE_DF_SCOPEWORD 12
+#define SCE_DF_OPERATOR 13
+#define SCE_DF_ICODE 14
 /* --Autogenerated -- end of section automatically generated from Scintilla.iface */
 
 #endif

scintilla/include/Scintilla.h

@@ -135,6 +135,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SC_MARK_UNDERLINE 29
 #define SC_MARK_RGBAIMAGE 30
 #define SC_MARK_BOOKMARK 31
+#define SC_MARK_VERTICALBOOKMARK 32
 #define SC_MARK_CHARACTER 10000
 #define SC_MARKNUM_FOLDEREND 25
 #define SC_MARKNUM_FOLDEROPENMID 26
@@ -266,6 +267,8 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_SETCARETPERIOD 2076
 #define SCI_SETWORDCHARS 2077
 #define SCI_GETWORDCHARS 2646
+#define SCI_SETCHARACTERCATEGORYOPTIMIZATION 2720
+#define SCI_GETCHARACTERCATEGORYOPTIMIZATION 2721
 #define SCI_BEGINUNDOACTION 2078
 #define SCI_ENDUNDOACTION 2079
 #define INDIC_PLAIN 0
@@ -290,14 +293,14 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define INDIC_POINTCHARACTER 19
 #define INDIC_GRADIENT 20
 #define INDIC_GRADIENTCENTRE 21
+#define INDIC_CONTAINER 8
 #define INDIC_IME 32
 #define INDIC_IME_MAX 35
 #define INDIC_MAX 35
-#define INDIC_CONTAINER 8
+#define INDICATOR_CONTAINER 8
-#define INDIC0_MASK 0x20
+#define INDICATOR_IME 32
-#define INDIC1_MASK 0x40
+#define INDICATOR_IME_MAX 35
-#define INDIC2_MASK 0x80
+#define INDICATOR_MAX 35
-#define INDICS_MASK 0xE0
 #define SCI_INDICSETSTYLE 2080
 #define SCI_INDICGETSTYLE 2081
 #define SCI_INDICSETFORE 2082
@@ -396,6 +399,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SC_PRINT_SCREENCOLOURS 5
 #define SCI_SETPRINTCOLOURMODE 2148
 #define SCI_GETPRINTCOLOURMODE 2149
+#define SCFIND_NONE 0x0
 #define SCFIND_WHOLEWORD 0x2
 #define SCFIND_MATCHCASE 0x4
 #define SCFIND_WORDSTART 0x00100000
@@ -490,6 +494,9 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SC_FOLDDISPLAYTEXT_STANDARD 1
 #define SC_FOLDDISPLAYTEXT_BOXED 2
 #define SCI_FOLDDISPLAYTEXTSETSTYLE 2701
+#define SCI_FOLDDISPLAYTEXTGETSTYLE 2707
+#define SCI_SETDEFAULTFOLDDISPLAYTEXT 2722
+#define SCI_GETDEFAULTFOLDDISPLAYTEXT 2723
 #define SC_FOLDACTION_CONTRACT 0
 #define SC_FOLDACTION_EXPAND 1
 #define SC_FOLDACTION_TOGGLE 2
@@ -828,7 +835,9 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define CARETSTYLE_LINE 1
 #define CARETSTYLE_BLOCK 2
 #define CARETSTYLE_OVERSTRIKE_BAR 0
-#define CARETSTYLE_OVERSTRIKE_BLOCK 16
+#define CARETSTYLE_OVERSTRIKE_BLOCK 0x10
+#define CARETSTYLE_INS_MASK 0xF
+#define CARETSTYLE_BLOCK_AFTER 0x100
 #define SCI_SETCARETSTYLE 2512
 #define SCI_GETCARETSTYLE 2513
 #define SCI_SETINDICATORCURRENT 2500
@@ -887,6 +896,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_ANNOTATIONGETSTYLEOFFSET 2551
 #define SCI_RELEASEALLEXTENDEDSTYLES 2552
 #define SCI_ALLOCATEEXTENDEDSTYLES 2553
+#define UNDO_NONE 0
 #define UNDO_MAY_COALESCE 1
 #define SCI_ADDUNDOACTION 2560
 #define SCI_CHARPOSITIONFROMPOINT 2561
@@ -1020,6 +1030,7 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SCI_NAMEOFSTYLE 4030
 #define SCI_TAGSOFSTYLE 4031
 #define SCI_DESCRIPTIONOFSTYLE 4032
+#define SC_MOD_NONE 0x0
 #define SC_MOD_INSERTTEXT 0x1
 #define SC_MOD_DELETETEXT 0x2
 #define SC_MOD_CHANGESTYLE 0x4
@@ -1082,6 +1093,9 @@ typedef sptr_t (*SciFnDirect)(sptr_t ptr, unsigned int iMessage, uptr_t wParam,
 #define SC_AC_TAB 3
 #define SC_AC_NEWLINE 4
 #define SC_AC_COMMAND 5
+#define SC_CHARACTERSOURCE_DIRECT_INPUT 0
+#define SC_CHARACTERSOURCE_TENTATIVE_INPUT 1
+#define SC_CHARACTERSOURCE_IME_RESULT 2
 #define SCN_STYLENEEDED 2000
 #define SCN_CHARADDED 2001
 #define SCN_SAVEPOINTREACHED 2002
@@ -1228,6 +1242,7 @@ struct SCNotification {
 	int updated;	/* SCN_UPDATEUI */
 	int listCompletionMethod;
 	/* SCN_AUTOCSELECTION, SCN_AUTOCCOMPLETED, SCN_USERLISTSELECTION, */
+	int characterSource;	/* SCN_CHARADDED */
 };
 
 struct SearchResultMarking {
@@ -1258,6 +1273,11 @@ struct SearchResultMarkings {
 #define SCI_GETSTYLEBITS 2091
 #define SCI_GETSTYLEBITSNEEDED 4011
 
+#define INDIC0_MASK 0x20
+#define INDIC1_MASK 0x40
+#define INDIC2_MASK 0x80
+#define INDICS_MASK 0xE0
+
 #endif
 
 #endif

scintilla/lexers/LexCIL.cxx

@@ -28,7 +28,6 @@
 #include "Accessor.h"
 #include "StyleContext.h"
 #include "CharacterSet.h"
-#include "CharacterCategory.h"
 #include "LexerModule.h"
 #include "OptionSet.h"
 #include "DefaultLexer.h"

scintilla/lexers/LexCPP.cxx

@@ -18,6 +18,7 @@
 #include <vector>
 #include <map>
 #include <algorithm>
+#include <iterator>
 
 #include "ILexer.h"
 #include "Scintilla.h"
@@ -39,7 +40,7 @@ using namespace Scintilla;
 namespace {
 	// Use an unnamed namespace to protect the functions and classes from name conflicts
 
-bool IsSpaceEquiv(int state) noexcept {
+constexpr bool IsSpaceEquiv(int state) noexcept {
 	return (state <= SCE_C_COMMENTDOC) ||
 		// including SCE_C_DEFAULT, SCE_C_COMMENT, SCE_C_COMMENTLINE
 		(state == SCE_C_COMMENTLINEDOC) || (state == SCE_C_COMMENTDOCKEYWORD) ||
@@ -86,7 +87,7 @@ bool followsReturnKeyword(const StyleContext &sc, LexAccessor &styler) {
 	return !*s;
 }
 
-bool IsSpaceOrTab(int ch) noexcept {
+constexpr bool IsSpaceOrTab(int ch) noexcept {
 	return ch == ' ' || ch == '\t';
 }
 
@@ -145,7 +146,7 @@ void highlightTaskMarker(StyleContext &sc, LexAccessor &styler,
 	if ((isoperator(sc.chPrev) || IsASpace(sc.chPrev)) && markerList.Length()) {
 		const int lengthMarker = 50;
 		char marker[lengthMarker+1] = "";
-		const Sci_Position currPos = static_cast<Sci_Position>(sc.currentPos);
+		const Sci_Position currPos = sc.currentPos;
 		int i = 0;
 		while (i < lengthMarker) {
 			const char ch = styler.SafeGetCharAt(currPos + i);
@@ -173,7 +174,7 @@ struct EscapeSequence {
 	CharacterSet *escapeSetValid;
 	EscapeSequence() {
 		digitsLeft = 0;
-		escapeSetValid = 0;
+		escapeSetValid = nullptr;
 		setHexDigits = CharacterSet(CharacterSet::setDigits, "ABCDEFabcdef");
 		setOctDigits = CharacterSet(CharacterSet::setNone, "01234567");
 	}
@@ -226,7 +227,7 @@ std::string GetRestOfLine(LexAccessor &styler, Sci_Position start, bool allowSpa
 	return restOfLine;
 }
 
-bool IsStreamCommentStyle(int style) noexcept {
+constexpr bool IsStreamCommentStyle(int style) noexcept {
 	return style == SCE_C_COMMENT ||
 		style == SCE_C_COMMENTDOC ||
 		style == SCE_C_COMMENTDOCKEYWORD ||
@@ -244,12 +245,24 @@ struct PPDefinition {
 	}
 };
 
+const int inactiveFlag = 0x40;
+
 class LinePPState {
-	int state;
+	// Track the state of preprocessor conditionals to allow showing active and inactive
-	int ifTaken;
+	// code in different styles.
-	int level;
+	// Only works up to 31 levels of conditional nesting.
+
+	// state is a bit mask with 1 bit per level
+	// bit is 1 for level if section inactive, so any bits set = inactive style
+	int state = 0;
+	// ifTaken is a bit mask with 1 bit per level
+	// bit is 1 for level if some branch at this level has been taken
+	int ifTaken = 0;
+	// level is the nesting level of #if constructs
+	int level = -1;
+	static const int maximumNestingLevel = 31;
 	bool ValidLevel() const noexcept {
-		return level >= 0 && level < 32;
+		return level >= 0 && level < maximumNestingLevel;
 	}
 	int maskLevel() const noexcept {
 		if (level >= 0) {
@@ -259,11 +272,17 @@ class LinePPState {
 		}
 	}
 public:
-	LinePPState() : state(0), ifTaken(0), level(-1) {
+	LinePPState() noexcept {
+	}
+	bool IsActive() const noexcept {
+		return state == 0;
 	}
 	bool IsInactive() const noexcept {
 		return state != 0;
 	}
+	int ActiveState() const noexcept {
+		return state ? inactiveFlag : 0;
+	}
 	bool CurrentIfTaken() const noexcept {
 		return (ifTaken & maskLevel()) != 0;
 	}
@@ -369,7 +388,7 @@ const char *const cppWordLists[] = {
             "Global classes and typedefs",
             "Preprocessor definitions",
             "Task marker and error marker keywords",
-            0,
+            nullptr,
 };
 
 struct OptionSetCPP : public OptionSet<OptionsCPP> {
@@ -478,6 +497,8 @@ LexicalClass lexicalClasses[] = {
 	27, "SCE_C_ESCAPESEQUENCE", "literal string escapesequence", "Escape sequence",
 };
 
+const int sizeLexicalClasses = static_cast<int>(std::size(lexicalClasses));
+
 }
 
 class LexerCPP : public ILexer4 {
@@ -500,7 +521,8 @@ class LexerCPP : public ILexer4 {
 	struct SymbolValue {
 		std::string value;
 		std::string arguments;
-		SymbolValue(const std::string &value_="", const std::string &arguments_="") : value(value_), arguments(arguments_) {
+		SymbolValue() noexcept = default;
+		SymbolValue(const std::string &value_, const std::string &arguments_) : value(value_), arguments(arguments_) {
 		}
 		SymbolValue &operator = (const std::string &value_) {
 			value = value_;
@@ -517,7 +539,6 @@ class LexerCPP : public ILexer4 {
 	OptionSetCPP osCPP;
 	EscapeSequence escapeSeq;
 	SparseState<std::string> rawStringTerminators;
-	enum { activeFlag = 0x40 };
 	enum { ssIdentifier, ssDocKeyword };
 	SubStyles subStyles;
 	std::string returnBuffer;
@@ -530,14 +551,19 @@ public:
 		setMultOp(CharacterSet::setNone, "*/%"),
 		setRelOp(CharacterSet::setNone, "=!<>"),
 		setLogicalOp(CharacterSet::setNone, "|&"),
-		subStyles(styleSubable, 0x80, 0x40, activeFlag) {
+		subStyles(styleSubable, 0x80, 0x40, inactiveFlag) {
 	}
+	// Deleted so LexerCPP objects can not be copied.
+	LexerCPP(const LexerCPP &) = delete;
+	LexerCPP(LexerCPP &&) = delete;
+	void operator=(const LexerCPP &) = delete;
+	void operator=(LexerCPP &&) = delete;
 	virtual ~LexerCPP() {
 	}
-	void SCI_METHOD Release() override {
+	void SCI_METHOD Release() noexcept override {
 		delete this;
 	}
-	int SCI_METHOD Version() const override {
+	int SCI_METHOD Version() const noexcept override {
 		return lvRelease4;
 	}
 	const char * SCI_METHOD PropertyNames() override {
@@ -557,11 +583,11 @@ public:
 	void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
 	void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
 
-	void * SCI_METHOD PrivateCall(int, void *) override {
+	void * SCI_METHOD PrivateCall(int, void *) noexcept override {
-		return 0;
+		return nullptr;
 	}
 
-	int SCI_METHOD LineEndTypesSupported() override {
+	int SCI_METHOD LineEndTypesSupported() noexcept override {
 		return SC_LINE_END_TYPE_UNICODE;
 	}
 
@@ -576,10 +602,10 @@ public:
 	}
 	int SCI_METHOD StyleFromSubStyle(int subStyle) override {
 		const int styleBase = subStyles.BaseStyle(MaskActive(subStyle));
-		const int active = subStyle & activeFlag;
+		const int inactive = subStyle & inactiveFlag;
-		return styleBase | active;
+		return styleBase | inactive;
 	}
-	int SCI_METHOD PrimaryStyleFromStyle(int style) override {
+	int SCI_METHOD PrimaryStyleFromStyle(int style) noexcept override {
 		return MaskActive(style);
 	}
 	void SCI_METHOD FreeSubStyles() override {
@@ -588,21 +614,21 @@ public:
 	void SCI_METHOD SetIdentifiers(int style, const char *identifiers) override {
 		subStyles.SetIdentifiers(style, identifiers);
 	}
-	int SCI_METHOD DistanceToSecondaryStyles() override {
+	int SCI_METHOD DistanceToSecondaryStyles() noexcept override {
-		return activeFlag;
+		return inactiveFlag;
 	}
-	const char * SCI_METHOD GetSubStyleBases() override {
+	const char * SCI_METHOD GetSubStyleBases() noexcept override {
 		return styleSubable;
 	}
 	int SCI_METHOD NamedStyles() override {
 		return std::max(subStyles.LastAllocated() + 1,
-			static_cast<int>(ELEMENTS(lexicalClasses))) +
+			sizeLexicalClasses) +
-			activeFlag;
+			inactiveFlag;
 	}
 	const char * SCI_METHOD NameOfStyle(int style) override {
 		if (style >= NamedStyles())
 			return "";
-		if (style < static_cast<int>(ELEMENTS(lexicalClasses)))
+		if (style < sizeLexicalClasses)
 			return lexicalClasses[style].name;
 		// TODO: inactive and substyles
 		return "";
@@ -615,23 +641,23 @@ public:
 		if (firstSubStyle >= 0) {
 			const int lastSubStyle = subStyles.LastAllocated();
 			if (((style >= firstSubStyle) && (style <= (lastSubStyle))) ||
-				((style >= firstSubStyle + activeFlag) && (style <= (lastSubStyle + activeFlag)))) {
+				((style >= firstSubStyle + inactiveFlag) && (style <= (lastSubStyle + inactiveFlag)))) {
 				int styleActive = style;
 				if (style > lastSubStyle) {
 					returnBuffer = "inactive ";
-					styleActive -= activeFlag;
+					styleActive -= inactiveFlag;
 				}
 				const int styleMain = StyleFromSubStyle(styleActive);
 				returnBuffer += lexicalClasses[styleMain].tags;
 				return returnBuffer.c_str();
 			}
 		}
-		if (style < static_cast<int>(ELEMENTS(lexicalClasses)))
+		if (style < sizeLexicalClasses)
 			return lexicalClasses[style].tags;
-		if (style >= activeFlag) {
+		if (style >= inactiveFlag) {
 			returnBuffer = "inactive ";
-			const int styleActive = style - activeFlag;
+			const int styleActive = style - inactiveFlag;
-			if (styleActive < static_cast<int>(ELEMENTS(lexicalClasses)))
+			if (styleActive < sizeLexicalClasses)
 				returnBuffer += lexicalClasses[styleActive].tags;
 			else
 				returnBuffer = "";
@@ -642,7 +668,7 @@ public:
 	const char * SCI_METHOD DescriptionOfStyle(int style) override {
 		if (style >= NamedStyles())
 			return "";
-		if (style < static_cast<int>(ELEMENTS(lexicalClasses)))
+		if (style < sizeLexicalClasses)
 			return lexicalClasses[style].description;
 		// TODO: inactive and substyles
 		return "";
@@ -654,8 +680,8 @@ public:
 	static ILexer4 *LexerFactoryCPPInsensitive() {
 		return new LexerCPP(false);
 	}
-	static int MaskActive(int style) noexcept {
+	constexpr static int MaskActive(int style) noexcept {
-		return style & ~activeFlag;
+		return style & ~inactiveFlag;
 	}
 	void EvaluateTokens(std::vector<std::string> &tokens, const SymbolTable &preprocessorDefinitions);
 	std::vector<std::string> Tokenize(const std::string &expr) const;
@@ -676,7 +702,7 @@ Sci_Position SCI_METHOD LexerCPP::PropertySet(const char *key, const char *val)
 }
 
 Sci_Position SCI_METHOD LexerCPP::WordListSet(int n, const char *wl) {
-	WordList *wordListN = 0;
+	WordList *wordListN = nullptr;
 	switch (n) {
 	case 0:
 		wordListN = &keywords;
@@ -813,12 +839,12 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 	std::string rawStringTerminator = rawStringTerminators.ValueAt(lineCurrent-1);
 	SparseState<std::string> rawSTNew(lineCurrent);
 
-	int activitySet = preproc.IsInactive() ? activeFlag : 0;
+	int activitySet = preproc.ActiveState();
 
 	const WordClassifier &classifierIdentifiers = subStyles.Classifier(SCE_C_IDENTIFIER);
 	const WordClassifier &classifierDocKeyWords = subStyles.Classifier(SCE_C_COMMENTDOCKEYWORD);
 
-	Sci_Position lineEndNext = styler.LineEnd(lineCurrent);
+	Sci_PositionU lineEndNext = styler.LineEnd(lineCurrent);
 
 	for (; sc.More();) {
 
@@ -840,7 +866,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 			isIncludePreprocessor = false;
 			inRERange = false;
 			if (preproc.IsInactive()) {
-				activitySet = activeFlag;
+				activitySet = inactiveFlag;
 				sc.SetState(sc.state | activitySet);
 			}
 		}
@@ -856,7 +882,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 
 		// Handle line continuation generically.
 		if (sc.ch == '\\') {
-			if (static_cast<Sci_Position>((sc.currentPos+1)) >= lineEndNext) {
+			if ((sc.currentPos+1) >= lineEndNext) {
 				lineCurrent++;
 				lineEndNext = styler.LineEnd(lineCurrent);
 				vlls.Add(lineCurrent, preproc);
@@ -1128,7 +1154,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 					while ((sc.ch < 0x80) && islower(sc.ch))
 						sc.Forward();    // gobble regex flags
 					sc.SetState(SCE_C_DEFAULT|activitySet);
-				} else if (sc.ch == '\\' && (static_cast<Sci_Position>(sc.currentPos+1) < lineEndNext)) {
+				} else if (sc.ch == '\\' && ((sc.currentPos+1) < lineEndNext)) {
 					// Gobble up the escaped character
 					sc.Forward();
 				} else if (sc.ch == '[') {
@@ -1265,6 +1291,8 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 					isIncludePreprocessor = true;
 				} else {
 					if (options.trackPreprocessor) {
+						// If #if is nested too deeply (>31 levels) the active/inactive appearance
+						// will stop reflecting the code.
 						if (sc.Match("ifdef") || sc.Match("ifndef")) {
 							const bool isIfDef = sc.Match("ifdef");
 							const int startRest = isIfDef ? 5 : 6;
@@ -1276,47 +1304,58 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 							const bool ifGood = EvaluateExpression(restOfLine, preprocessorDefinitions);
 							preproc.StartSection(ifGood);
 						} else if (sc.Match("else")) {
+							// #else is shown as active if either preceding or following section is active
+							// as that means that it contributed to the result.
 							if (!preproc.CurrentIfTaken()) {
+								// Inactive, may become active if parent scope active
+								assert(sc.state == (SCE_C_PREPROCESSOR|inactiveFlag));
 								preproc.InvertCurrentLevel();
-								activitySet = preproc.IsInactive() ? activeFlag : 0;
+								activitySet = preproc.ActiveState();
+								// If following is active then show "else" as active
 								if (!activitySet)
-									sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
+									sc.ChangeState(SCE_C_PREPROCESSOR);
-							} else if (!preproc.IsInactive()) {
+							} else if (preproc.IsActive()) {
+								// Active -> inactive
+								assert(sc.state == SCE_C_PREPROCESSOR);
 								preproc.InvertCurrentLevel();
-								activitySet = preproc.IsInactive() ? activeFlag : 0;
+								activitySet = preproc.ActiveState();
-								if (!activitySet)
+								// Continue to show "else" as active as it ends active section.
-									sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
 							}
 						} else if (sc.Match("elif")) {
 							// Ensure only one chosen out of #if .. #elif .. #elif .. #else .. #endif
+							// #elif is shown as active if either preceding or following section is active
+							// as that means that it contributed to the result.
 							if (!preproc.CurrentIfTaken()) {
+								// Inactive, if expression true then may become active if parent scope active
+								assert(sc.state == (SCE_C_PREPROCESSOR|inactiveFlag));
 								// Similar to #if
 								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 4, true);
 								const bool ifGood = EvaluateExpression(restOfLine, preprocessorDefinitions);
 								if (ifGood) {
 									preproc.InvertCurrentLevel();
-									activitySet = preproc.IsInactive() ? activeFlag : 0;
+									activitySet = preproc.ActiveState();
 									if (!activitySet)
-										sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
+										sc.ChangeState(SCE_C_PREPROCESSOR);
 								}
-							} else if (!preproc.IsInactive()) {
+							} else if (preproc.IsActive()) {
+								// Active -> inactive
+								assert(sc.state == SCE_C_PREPROCESSOR);
 								preproc.InvertCurrentLevel();
-								activitySet = preproc.IsInactive() ? activeFlag : 0;
+								activitySet = preproc.ActiveState();
-								if (!activitySet)
+								// Continue to show "elif" as active as it ends active section.
-									sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
 							}
 						} else if (sc.Match("endif")) {
 							preproc.EndSection();
-							activitySet = preproc.IsInactive() ? activeFlag : 0;
+							activitySet = preproc.ActiveState();
 							sc.ChangeState(SCE_C_PREPROCESSOR|activitySet);
 						} else if (sc.Match("define")) {
-							if (options.updatePreprocessor && !preproc.IsInactive()) {
+							if (options.updatePreprocessor && preproc.IsActive()) {
 								std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 6, true);
 								size_t startName = 0;
 								while ((startName < restOfLine.length()) && IsSpaceOrTab(restOfLine[startName]))
 									startName++;
 								size_t endName = startName;
-								while ((endName < restOfLine.length()) && setWord.Contains(static_cast<unsigned char>(restOfLine[endName])))
+								while ((endName < restOfLine.length()) && setWord.Contains(restOfLine[endName]))
 									endName++;
 								std::string key = restOfLine.substr(startName, endName-startName);
 								if ((endName < restOfLine.length()) && (restOfLine.at(endName) == '(')) {
@@ -1348,7 +1387,7 @@ void SCI_METHOD LexerCPP::Lex(Sci_PositionU startPos, Sci_Position length, int i
 								}
 							}
 						} else if (sc.Match("undef")) {
-							if (options.updatePreprocessor && !preproc.IsInactive()) {
+							if (options.updatePreprocessor && preproc.IsActive()) {
 								const std::string restOfLine = GetRestOfLine(styler, sc.currentPos + 5, false);
 								std::vector<std::string> tokens = Tokenize(restOfLine);
 								if (tokens.size() >= 1) {
@@ -1542,7 +1581,7 @@ void LexerCPP::EvaluateTokens(std::vector<std::string> &tokens, const SymbolTabl
 	size_t iterations = 0;	// Limit number of iterations in case there is a recursive macro.
 	for (size_t i = 0; (i<tokens.size()) && (iterations < maxIterations);) {
 		iterations++;
-		if (setWordStart.Contains(static_cast<unsigned char>(tokens[i][0]))) {
+		if (setWordStart.Contains(tokens[i][0])) {
 			SymbolTable::const_iterator it = preprocessorDefinitions.find(tokens[i]);
 			if (it != preprocessorDefinitions.end()) {
 				// Tokenize value
@@ -1569,7 +1608,7 @@ void LexerCPP::EvaluateTokens(std::vector<std::string> &tokens, const SymbolTabl
 						macroTokens.erase(std::remove_if(macroTokens.begin(), macroTokens.end(), OnlySpaceOrTab), macroTokens.end());
 
 						for (size_t iMacro = 0; iMacro < macroTokens.size();) {
-							if (setWordStart.Contains(static_cast<unsigned char>(macroTokens[iMacro][0]))) {
+							if (setWordStart.Contains(macroTokens[iMacro][0])) {
 								std::map<std::string, std::string>::const_iterator itFind = arguments.find(macroTokens[iMacro]);
 								if (itFind != arguments.end()) {
 									// TODO: Possible that value will be expression so should insert tokenized form
@@ -1687,9 +1726,9 @@ std::vector<std::string> LexerCPP::Tokenize(const std::string &expr) const {
 	const char *cp = expr.c_str();
 	while (*cp) {
 		std::string word;
-		if (setWord.Contains(static_cast<unsigned char>(*cp))) {
+		if (setWord.Contains(*cp)) {
 			// Identifiers and numbers
-			while (setWord.Contains(static_cast<unsigned char>(*cp))) {
+			while (setWord.Contains(*cp)) {
 				word += *cp;
 				cp++;
 			}
@@ -1698,17 +1737,17 @@ std::vector<std::string> LexerCPP::Tokenize(const std::string &expr) const {
 				word += *cp;
 				cp++;
 			}
-		} else if (setRelOp.Contains(static_cast<unsigned char>(*cp))) {
+		} else if (setRelOp.Contains(*cp)) {
 			word += *cp;
 			cp++;
-			if (setRelOp.Contains(static_cast<unsigned char>(*cp))) {
+			if (setRelOp.Contains(*cp)) {
 				word += *cp;
 				cp++;
 			}
-		} else if (setLogicalOp.Contains(static_cast<unsigned char>(*cp))) {
+		} else if (setLogicalOp.Contains(*cp)) {
 			word += *cp;
 			cp++;
-			if (setLogicalOp.Contains(static_cast<unsigned char>(*cp))) {
+			if (setLogicalOp.Contains(*cp)) {
 				word += *cp;
 				cp++;
 			}

scintilla/lexers/LexCmake.cxx

@@ -85,10 +85,11 @@ static int calculateFoldCmake(Sci_PositionU start, Sci_PositionU end, int foldle
 
     if ( CompareCaseInsensitive(s, "IF") == 0 || CompareCaseInsensitive(s, "WHILE") == 0
          || CompareCaseInsensitive(s, "MACRO") == 0 || CompareCaseInsensitive(s, "FOREACH") == 0
-         || CompareCaseInsensitive(s, "ELSEIF") == 0 )
+         || CompareCaseInsensitive(s, "FUNCTION") == 0 || CompareCaseInsensitive(s, "ELSEIF") == 0)
         newFoldlevel++;
     else if ( CompareCaseInsensitive(s, "ENDIF") == 0 || CompareCaseInsensitive(s, "ENDWHILE") == 0
-              || CompareCaseInsensitive(s, "ENDMACRO") == 0 || CompareCaseInsensitive(s, "ENDFOREACH") == 0)
+              || CompareCaseInsensitive(s, "ENDMACRO") == 0 || CompareCaseInsensitive(s, "ENDFOREACH") == 0
+              || CompareCaseInsensitive(s, "ENDFUNCTION") == 0)
         newFoldlevel--;
     else if ( bElse && CompareCaseInsensitive(s, "ELSEIF") == 0 )
         newFoldlevel++;

scintilla/lexers/LexDataflex.cxx

@@ -0,0 +1,608 @@
+// Scintilla source code edit control
+/** @file LexDataflex.cxx
+ ** Lexer for DataFlex.
+ ** Based on LexPascal.cxx
+ ** Written by Wil van Antwerpen, June 2019
+ **/
+
+/*
+// The License.txt file describes the conditions under which this software may be distributed.
+
+A few words about features of LexDataflex...
+
+Generally speaking LexDataflex tries to support all available DataFlex features (up
+to DataFlex 19.1 at this time).
+
+~ FOLDING:
+
+Folding is supported in the following cases:
+
+- Folding of stream-like comments
+- Folding of groups of consecutive line comments
+- Folding of preprocessor blocks (the following preprocessor blocks are
+supported: #IFDEF, #IFNDEF, #ENDIF and #HEADER / #ENDHEADER 
+blocks), 
+- Folding of code blocks on appropriate keywords (the following code blocks are
+supported: "begin, struct, type, case / end" blocks, class & object
+declarations and interface declarations)
+
+Remarks:
+
+- We pass 4 arrays to the lexer:
+1. The DataFlex keyword list, these are normal DataFlex keywords
+2. The Scope Open list, for example, begin / procedure / while
+3. The Scope Close list, for example, end / end_procedure / loop
+4. Operator list, for ex. + / - / * / Lt / iand
+These lists are all mutually exclusive, scope open words should not be in the keyword list and vice versa
+
+- Folding of code blocks tries to handle all special cases in which folding
+should not occur. 
+
+~ KEYWORDS:
+
+The list of keywords that can be used in dataflex.properties file (up to DataFlex
+19.1):
+
+- Keywords: .. snipped .. see dataflex.properties file.
+
+*/
+
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdarg.h>
+#include <assert.h>
+#include <ctype.h>
+
+#include "ILexer.h"
+#include "Scintilla.h"
+#include "SciLexer.h"
+
+#include "WordList.h"
+#include "LexAccessor.h"
+#include "Accessor.h"
+#include "StyleContext.h"
+#include "CharacterSet.h"
+#include "LexerModule.h"
+
+using namespace Scintilla;
+
+
+static void GetRangeLowered(Sci_PositionU start,
+		Sci_PositionU end,
+		Accessor &styler,
+		char *s,
+		Sci_PositionU len) {
+	Sci_PositionU i = 0;
+	while ((i < end - start + 1) && (i < len-1)) {
+		s[i] = static_cast<char>(tolower(styler[start + i]));
+		i++;
+	}
+	s[i] = '\0';
+}
+
+static void GetForwardRangeLowered(Sci_PositionU start,
+		CharacterSet &charSet,
+		Accessor &styler,
+		char *s,
+		Sci_PositionU len) {
+	Sci_PositionU i = 0;
+	while ((i < len-1) && charSet.Contains(styler.SafeGetCharAt(start + i))) {
+		s[i] = static_cast<char>(tolower(styler.SafeGetCharAt(start + i)));
+		i++;
+	}
+	s[i] = '\0';
+
+}
+
+enum {
+	stateInICode = 0x1000,
+	stateSingleQuoteOpen = 0x2000,
+	stateDoubleQuoteOpen = 0x4000,
+	stateFoldInPreprocessor = 0x0100,
+	stateFoldInCaseStatement = 0x0200,
+	stateFoldInPreprocessorLevelMask = 0x00FF,
+	stateFoldMaskAll = 0x0FFF
+};
+
+
+static bool IsFirstDataFlexWord(Sci_Position pos, Accessor &styler) {
+	Sci_Position line = styler.GetLine(pos);
+	Sci_Position start_pos = styler.LineStart(line);
+	for (Sci_Position i = start_pos; i < pos; i++) {
+		char ch = styler.SafeGetCharAt(i);
+		if (!(ch == ' ' || ch == '\t'))
+			return false;
+	}
+	return true;
+}
+
+
+inline bool IsADataFlexField(int ch) {
+	return (ch == '.');
+}
+
+
+static void ClassifyDataFlexWord(WordList *keywordlists[], StyleContext &sc, Accessor &styler) {
+	WordList& keywords = *keywordlists[0];
+	WordList& scopeOpen   = *keywordlists[1];
+	WordList& scopeClosed = *keywordlists[2];
+	WordList& operators   = *keywordlists[3];
+
+	char s[100];
+	int oldState;
+	int newState;
+	size_t tokenlen;
+
+    oldState = sc.state;
+	newState = oldState;
+	sc.GetCurrentLowered(s, sizeof(s));
+	tokenlen = strnlen(s,sizeof(s));
+	if (keywords.InList(s)) {
+		// keywords in DataFlex can be used as table column names (file.field) and as such they
+		// should not be characterized as a keyword. So test for that.
+		// for ex. somebody using date as field name.
+        if (!IsADataFlexField(sc.GetRelative(-static_cast<int>(tokenlen+1)))) {
+	      newState = SCE_DF_WORD;
+	    }
+	}
+	if (oldState == newState) {
+		if ((scopeOpen.InList(s) || scopeClosed.InList(s)) && (strcmp(s, "for") != 0) && (strcmp(s, "repeat") != 0)) {
+			// scope words in DataFlex can be used as table column names (file.field) and as such they
+			// should not be characterized as a scope word. So test for that.
+			// for ex. somebody using procedure for field name.
+			if (!IsADataFlexField(sc.GetRelative(-static_cast<int>(tokenlen+1)))) {
+				newState = SCE_DF_SCOPEWORD;
+			}
+		} 
+		// no code folding on the next words, but just want to paint them like keywords (as they are) (??? doesn't the code to the opposite?)
+		if (strcmp(s, "if") == 0 ||  
+			strcmp(s, "ifnot") == 0 ||
+			strcmp(s, "case") == 0 ||
+			strcmp(s, "else") == 0 ) {
+				newState = SCE_DF_SCOPEWORD;
+		} 
+	}
+	if (oldState != newState && newState == SCE_DF_WORD) {
+		// a for loop must have for at the start of the line, for is also used in "define abc for 123"
+		if ( (strcmp(s, "for") == 0) && (IsFirstDataFlexWord(sc.currentPos-3, styler)) ) {   
+				newState = SCE_DF_SCOPEWORD;
+		} 
+	}
+	if (oldState != newState && newState == SCE_DF_WORD) {
+		// a repeat loop must have repeat at the start of the line, repeat is also used in 'move (repeat("d",5)) to sFoo'
+		if ( (strcmp(s, "repeat") == 0) && (IsFirstDataFlexWord(sc.currentPos-6, styler)) ) {   
+				newState = SCE_DF_SCOPEWORD;
+		} 
+	}
+	if (oldState == newState)  {
+	  if (operators.InList(s)) {
+		  newState = SCE_DF_OPERATOR;
+		} 
+	}
+
+	if (oldState != newState) {
+		sc.ChangeState(newState);
+	}
+	sc.SetState(SCE_DF_DEFAULT);
+}
+
+static void ColouriseDataFlexDoc(Sci_PositionU startPos, Sci_Position length, int initStyle, WordList *keywordlists[],
+		Accessor &styler) {
+//	bool bSmartHighlighting = styler.GetPropertyInt("lexer.dataflex.smart.highlighting", 1) != 0;
+
+			CharacterSet setWordStart(CharacterSet::setAlpha, "_$#@", 0x80, true);
+			CharacterSet setWord(CharacterSet::setAlphaNum, "_$#@", 0x80, true);
+	CharacterSet setNumber(CharacterSet::setDigits, ".-+eE");
+	CharacterSet setHexNumber(CharacterSet::setDigits, "abcdefABCDEF");
+	CharacterSet setOperator(CharacterSet::setNone, "*+-/<=>^");
+
+	Sci_Position curLine = styler.GetLine(startPos);
+	int curLineState = curLine > 0 ? styler.GetLineState(curLine - 1) : 0;
+
+	StyleContext sc(startPos, length, initStyle, styler);
+
+	for (; sc.More(); sc.Forward()) {
+		if (sc.atLineEnd) {
+			// Update the line state, so it can be seen by next line
+			curLine = styler.GetLine(sc.currentPos);
+			styler.SetLineState(curLine, curLineState);
+		}
+
+		// Determine if the current state should terminate.
+		switch (sc.state) {
+			case SCE_DF_NUMBER:
+				if (!setNumber.Contains(sc.ch) || (sc.ch == '.' && sc.chNext == '.')) {
+					sc.SetState(SCE_DF_DEFAULT);
+				} else if (sc.ch == '-' || sc.ch == '+') {
+					if (sc.chPrev != 'E' && sc.chPrev != 'e') {
+						sc.SetState(SCE_DF_DEFAULT);
+					}
+				}
+				break;
+			case SCE_DF_IDENTIFIER:
+				if (!setWord.Contains(sc.ch)) {
+					ClassifyDataFlexWord(keywordlists, sc, styler);
+				}
+				break;
+			case SCE_DF_HEXNUMBER:
+				if (!(setHexNumber.Contains(sc.ch) || sc.ch == 'I') ) { // in |CI$22a we also want to color the "I"
+					sc.SetState(SCE_DF_DEFAULT);
+				}
+				break;
+			case SCE_DF_METATAG:
+				if (sc.atLineStart || sc.chPrev == '}') {
+					sc.SetState(SCE_DF_DEFAULT);
+				}
+				break;
+			case SCE_DF_PREPROCESSOR:
+				if (sc.atLineStart || IsASpaceOrTab(sc.ch)) {
+					sc.SetState(SCE_DF_DEFAULT);
+				}
+				break;
+			case SCE_DF_IMAGE:
+				if (sc.atLineStart && sc.Match("/*")) {
+					sc.Forward();  // these characters are still part of the DF Image
+					sc.ForwardSetState(SCE_DF_DEFAULT);
+				}
+				break;
+			case SCE_DF_PREPROCESSOR2:
+				// we don't have inline comments or preprocessor2 commands
+				//if (sc.Match('*', ')')) {
+				//	sc.Forward();
+				//	sc.ForwardSetState(SCE_DF_DEFAULT);
+				//}
+				break;
+			case SCE_DF_COMMENTLINE:
+				if (sc.atLineStart) {
+					sc.SetState(SCE_DF_DEFAULT);
+				}
+				break;
+			case SCE_DF_STRING:
+				if (sc.atLineEnd) {
+					sc.ChangeState(SCE_DF_STRINGEOL);
+				} else if (sc.ch == '\'' && sc.chNext == '\'') {
+					sc.Forward();
+				} else if (sc.ch == '\"' && sc.chNext == '\"') {
+					sc.Forward();
+				} else if (sc.ch == '\'' || sc.ch == '\"') {
+					if (sc.ch == '\'' && (curLineState & stateSingleQuoteOpen) ) {
+ 				    curLineState &= ~(stateSingleQuoteOpen);
+					sc.ForwardSetState(SCE_DF_DEFAULT);
+					}
+					else if (sc.ch == '\"' && (curLineState & stateDoubleQuoteOpen) ) {
+ 				    curLineState &= ~(stateDoubleQuoteOpen);
+					sc.ForwardSetState(SCE_DF_DEFAULT);
+					}
+				}
+				break;
+			case SCE_DF_STRINGEOL:
+				if (sc.atLineStart) {
+					sc.SetState(SCE_DF_DEFAULT);
+				}
+				break;
+			case SCE_DF_SCOPEWORD:
+				//if (!setHexNumber.Contains(sc.ch) && sc.ch != '$') {
+				//	sc.SetState(SCE_DF_DEFAULT);
+				//}
+				break;
+			case SCE_DF_OPERATOR:
+//				if (bSmartHighlighting && sc.chPrev == ';') {
+//					curLineState &= ~(stateInProperty | stateInExport);
+//				}
+				sc.SetState(SCE_DF_DEFAULT);
+				break;
+			case SCE_DF_ICODE:
+				if (sc.atLineStart || IsASpace(sc.ch) || isoperator(sc.ch)) {
+				sc.SetState(SCE_DF_DEFAULT);
+				}
+				break;
+		}
+
+		// Determine if a new state should be entered.
+		if (sc.state == SCE_DF_DEFAULT) {
+			if (IsADigit(sc.ch)) {
+				sc.SetState(SCE_DF_NUMBER);
+			} else if (sc.Match('/', '/') || sc.Match("#REM")) {
+				sc.SetState(SCE_DF_COMMENTLINE);
+			} else if ((sc.ch == '#' && !sc.Match("#REM")) && IsFirstDataFlexWord(sc.currentPos, styler)) {
+				sc.SetState(SCE_DF_PREPROCESSOR);
+			// || (sc.ch == '|' && sc.chNext == 'C' && sc.GetRelativeCharacter(2) == 'I' && sc.GetRelativeCharacter(3) == '$') ) {
+			} else if ((sc.ch == '$' && ((!setWord.Contains(sc.chPrev)) || sc.chPrev == 'I' ) ) || (sc.Match("|CI$")) ) {
+				sc.SetState(SCE_DF_HEXNUMBER); // start with $ and previous character not in a..zA..Z0..9 excluding "I" OR start with |CI$
+			} else if (setWordStart.Contains(sc.ch)) {
+				sc.SetState(SCE_DF_IDENTIFIER);
+			} else if (sc.ch == '{') {
+				sc.SetState(SCE_DF_METATAG);
+			//} else if (sc.Match("(*$")) {
+			//	sc.SetState(SCE_DF_PREPROCESSOR2);
+			} else if (sc.ch == '/' && setWord.Contains(sc.chNext) &&  sc.atLineStart) {
+				sc.SetState(SCE_DF_IMAGE);
+			//	sc.Forward();	// Eat the * so it isn't used for the end of the comment
+			} else if (sc.ch == '\'' || sc.ch == '\"') {
+				if (sc.ch == '\'' && !(curLineState & stateDoubleQuoteOpen)) {
+				  curLineState |= stateSingleQuoteOpen;
+				} else if (sc.ch == '\"' && !(curLineState & stateSingleQuoteOpen)) {
+				  curLineState |= stateDoubleQuoteOpen;
+				}
+			    sc.SetState(SCE_DF_STRING);
+			} else if (setOperator.Contains(sc.ch)) {
+				sc.SetState(SCE_DF_OPERATOR);
+//			} else if (curLineState & stateInICode) {
+				// ICode start ! in a string followed by close string mark is not icode
+			} else if ((sc.ch == '!') && !(sc.ch == '!' && ((sc.chNext == '\"') || (sc.ch == '\'')) )) {
+				sc.SetState(SCE_DF_ICODE);
+			}
+		}
+	}
+
+	if (sc.state == SCE_DF_IDENTIFIER && setWord.Contains(sc.chPrev)) {
+		ClassifyDataFlexWord(keywordlists, sc, styler);
+	}
+
+	sc.Complete();
+}
+
+static bool IsStreamCommentStyle(int style) {
+	return style == SCE_DF_IMAGE;
+}
+
+static bool IsCommentLine(Sci_Position line, Accessor &styler) {
+	Sci_Position pos = styler.LineStart(line);
+	Sci_Position eolPos = styler.LineStart(line + 1) - 1;
+	for (Sci_Position i = pos; i < eolPos; i++) {
+		char ch = styler[i];
+		char chNext = styler.SafeGetCharAt(i + 1);
+		int style = styler.StyleAt(i);
+		if (ch == '/' && chNext == '/' && style == SCE_DF_COMMENTLINE) {
+			return true;
+		} else if (!IsASpaceOrTab(ch)) {
+			return false;
+		}
+	}
+	return false;
+}
+
+
+
+static unsigned int GetFoldInPreprocessorLevelFlag(int lineFoldStateCurrent) {
+	return lineFoldStateCurrent & stateFoldInPreprocessorLevelMask;
+}
+
+static void SetFoldInPreprocessorLevelFlag(int &lineFoldStateCurrent, unsigned int nestLevel) {
+	lineFoldStateCurrent &= ~stateFoldInPreprocessorLevelMask;
+	lineFoldStateCurrent |= nestLevel & stateFoldInPreprocessorLevelMask;
+}
+
+static int ClassifyDataFlexPreprocessorFoldPoint(int &levelCurrent, int &lineFoldStateCurrent,
+		Sci_PositionU startPos, Accessor &styler) {
+	CharacterSet setWord(CharacterSet::setAlpha);
+
+	char s[100];	// Size of the longest possible keyword + one additional character + null
+	GetForwardRangeLowered(startPos, setWord, styler, s, sizeof(s));
+	size_t iLen = strnlen(s,sizeof(s));
+	size_t iWordSize = 0;
+
+	unsigned int nestLevel = GetFoldInPreprocessorLevelFlag(lineFoldStateCurrent);
+
+	if (strcmp(s, "command") == 0 ||
+		// The #if/#ifdef etcetera commands are not currently foldable as it is easy to write code that
+		// breaks the collaps logic, so we keep things simple and not include that for now.
+		strcmp(s, "header") == 0) {
+		nestLevel++;
+		SetFoldInPreprocessorLevelFlag(lineFoldStateCurrent, nestLevel);
+		lineFoldStateCurrent |= stateFoldInPreprocessor;
+		levelCurrent++;
+		iWordSize = iLen;
+	} else if (strcmp(s, "endcommand") == 0 ||
+		strcmp(s, "endheader") == 0) {
+		nestLevel--;
+		SetFoldInPreprocessorLevelFlag(lineFoldStateCurrent, nestLevel);
+		if (nestLevel == 0) {
+			lineFoldStateCurrent &= ~stateFoldInPreprocessor;
+		}
+		levelCurrent--;
+		iWordSize = iLen;
+		if (levelCurrent < SC_FOLDLEVELBASE) {
+			levelCurrent = SC_FOLDLEVELBASE;
+		}
+	}
+	return static_cast<int>(iWordSize);
+}
+
+
+static void ClassifyDataFlexWordFoldPoint(int &levelCurrent, int &lineFoldStateCurrent,
+		Sci_PositionU lastStart, Sci_PositionU currentPos, WordList *[], Accessor &styler) {
+	char s[100];
+
+	// property fold.dataflex.compilerlist
+	//	Set to 1 for enabling the code folding feature in *.prn files
+	bool foldPRN = styler.GetPropertyInt("fold.dataflex.compilerlist",0) != 0;
+
+	GetRangeLowered(lastStart, currentPos, styler, s, sizeof(s));
+
+	if (strcmp(s, "case") == 0) {
+		lineFoldStateCurrent |= stateFoldInCaseStatement;
+	} else if (strcmp(s, "begin") == 0) {
+		levelCurrent++;
+	} else if (strcmp(s, "for") == 0 ||
+		strcmp(s, "while") == 0 ||
+		strcmp(s, "repeat") == 0 ||
+		strcmp(s, "for_all") == 0 ||
+		strcmp(s, "struct") == 0 ||
+		strcmp(s, "type") == 0 ||
+		strcmp(s, "begin_row") == 0 ||
+		strcmp(s, "item_list") == 0 ||
+		strcmp(s, "begin_constraints") == 0 ||
+		strcmp(s, "begin_transaction") == 0 ||
+		strcmp(s, "enum_list") == 0 ||
+		strcmp(s, "class") == 0 || 
+		strcmp(s, "object") == 0 ||
+		strcmp(s, "cd_popup_object") == 0 ||
+		strcmp(s, "procedure") == 0 ||
+		strcmp(s, "procedure_section") == 0 ||
+		strcmp(s, "function") == 0 ) {
+			if ((IsFirstDataFlexWord(lastStart, styler )) || foldPRN) {
+			levelCurrent++;
+			}
+	} else if (strcmp(s, "end") == 0) {  // end is not always the first keyword, for example "case end"
+		levelCurrent--;
+		if (levelCurrent < SC_FOLDLEVELBASE) {
+			levelCurrent = SC_FOLDLEVELBASE;
+		}
+	} else if (strcmp(s, "loop") == 0 ||
+				   strcmp(s, "until") == 0 ||
+				   strcmp(s, "end_class") == 0 ||
+				   strcmp(s, "end_object") == 0 ||
+				   strcmp(s, "cd_end_object") == 0 ||
+				   strcmp(s, "end_procedure") == 0 ||
+				   strcmp(s, "end_function") == 0 ||
+				   strcmp(s, "end_for_all") == 0 ||
+				   strcmp(s, "end_struct") == 0 ||
+				   strcmp(s, "end_type") == 0 ||
+				   strcmp(s, "end_row") == 0 ||
+				   strcmp(s, "end_item_list") == 0 ||
+				   strcmp(s, "end_constraints") == 0 ||
+				   strcmp(s, "end_transaction") == 0 ||
+				   strcmp(s, "end_enum_list") == 0 ) {
+	//		lineFoldStateCurrent &= ~stateFoldInRecord;
+			if ((IsFirstDataFlexWord(lastStart, styler )) || foldPRN) {
+				levelCurrent--;
+				if (levelCurrent < SC_FOLDLEVELBASE) {
+					levelCurrent = SC_FOLDLEVELBASE;
+				}
+			}
+	}
+
+}
+
+
+static void ClassifyDataFlexMetaDataFoldPoint(int &levelCurrent, 
+		Sci_PositionU lastStart, Sci_PositionU currentPos, WordList *[], Accessor &styler) {
+	char s[100];
+
+	GetRangeLowered(lastStart, currentPos, styler, s, sizeof(s));
+
+    if (strcmp(s, "#beginsection") == 0) {
+		levelCurrent++;
+	} else if (strcmp(s, "#endsection") == 0) {
+			levelCurrent--;
+			if (levelCurrent < SC_FOLDLEVELBASE) {
+				levelCurrent = SC_FOLDLEVELBASE;
+			}
+	}
+
+}
+
+static void FoldDataFlexDoc(Sci_PositionU startPos, Sci_Position length, int initStyle, WordList *keywordlists[],
+		Accessor &styler) {
+	bool foldComment = styler.GetPropertyInt("fold.comment") != 0;
+	bool foldPreprocessor = styler.GetPropertyInt("fold.preprocessor") != 0;
+	bool foldCompact = styler.GetPropertyInt("fold.compact", 1) != 0;
+	Sci_PositionU endPos = startPos + length;
+	int visibleChars = 0;
+	Sci_Position lineCurrent = styler.GetLine(startPos);
+	int levelPrev = styler.LevelAt(lineCurrent) & SC_FOLDLEVELNUMBERMASK;
+	int levelCurrent = levelPrev;
+	int lineFoldStateCurrent = lineCurrent > 0 ? styler.GetLineState(lineCurrent - 1) & stateFoldMaskAll : 0;
+	char chNext = styler[startPos];
+	int styleNext = styler.StyleAt(startPos);
+	int style = initStyle;
+	int iWordSize;
+
+	Sci_Position lastStart = 0;
+	CharacterSet setWord(CharacterSet::setAlphaNum, "_$#@", 0x80, true);
+
+	for (Sci_PositionU i = startPos; i < endPos; i++) {
+		char ch = chNext;
+		chNext = styler.SafeGetCharAt(i + 1);
+		int stylePrev = style;
+		style = styleNext;
+		styleNext = styler.StyleAt(i + 1);
+		bool atEOL = (ch == '\r' && chNext != '\n') || (ch == '\n');
+
+		if (foldComment && IsStreamCommentStyle(style)) {
+			if (!IsStreamCommentStyle(stylePrev)) {
+				levelCurrent++;
+			} else if (!IsStreamCommentStyle(styleNext)) {
+				levelCurrent--;
+			}
+		}
+		if (foldComment && atEOL && IsCommentLine(lineCurrent, styler))
+		{
+			if (!IsCommentLine(lineCurrent - 1, styler)
+			    && IsCommentLine(lineCurrent + 1, styler))
+				levelCurrent++;
+			else if (IsCommentLine(lineCurrent - 1, styler)
+			         && !IsCommentLine(lineCurrent+1, styler))
+				levelCurrent--;
+		}
+		if (foldPreprocessor) {
+			if (style == SCE_DF_PREPROCESSOR) {
+				iWordSize = ClassifyDataFlexPreprocessorFoldPoint(levelCurrent, lineFoldStateCurrent, i + 1, styler);
+			//} else if (style == SCE_DF_PREPROCESSOR2 && ch == '(' && chNext == '*'
+			//           && styler.SafeGetCharAt(i + 2) == '$') {
+			//	ClassifyDataFlexPreprocessorFoldPoint(levelCurrent, lineFoldStateCurrent, i + 3, styler);
+				i = i + iWordSize;
+			}
+		}
+
+		if (stylePrev != SCE_DF_SCOPEWORD && style == SCE_DF_SCOPEWORD)
+		{
+			// Store last word start point.
+			lastStart = i;
+		}
+		if (stylePrev == SCE_DF_SCOPEWORD) {
+			if(setWord.Contains(ch) && !setWord.Contains(chNext)) {
+				ClassifyDataFlexWordFoldPoint(levelCurrent, lineFoldStateCurrent, lastStart, i, keywordlists, styler);
+			}
+		}
+
+		if (stylePrev == SCE_DF_METATAG && ch == '#')
+		{
+			// Store last word start point.
+			lastStart = i;
+		}
+		if (stylePrev == SCE_DF_METATAG) {
+			if(setWord.Contains(ch) && !setWord.Contains(chNext)) {
+				ClassifyDataFlexMetaDataFoldPoint(levelCurrent, lastStart, i, keywordlists, styler);
+			}
+		}
+
+		if (!IsASpace(ch))
+			visibleChars++;
+
+		if (atEOL) {
+			int lev = levelPrev;
+			if (visibleChars == 0 && foldCompact)
+				lev |= SC_FOLDLEVELWHITEFLAG;
+			if ((levelCurrent > levelPrev) && (visibleChars > 0))
+				lev |= SC_FOLDLEVELHEADERFLAG;
+			if (lev != styler.LevelAt(lineCurrent)) {
+				styler.SetLevel(lineCurrent, lev);
+			}
+			int newLineState = (styler.GetLineState(lineCurrent) & ~stateFoldMaskAll) | lineFoldStateCurrent;
+			styler.SetLineState(lineCurrent, newLineState);
+			lineCurrent++;
+			levelPrev = levelCurrent;
+			visibleChars = 0;
+		}
+	}
+
+	// If we didn't reach the EOL in previous loop, store line level and whitespace information.
+	// The rest will be filled in later...
+	int lev = levelPrev;
+	if (visibleChars == 0 && foldCompact)
+		lev |= SC_FOLDLEVELWHITEFLAG;
+	styler.SetLevel(lineCurrent, lev);
+}
+
+static const char * const dataflexWordListDesc[] = {
+	"Keywords",
+	"Scope open",
+	"Scope close",
+	"Operators",
+	0
+};
+
+LexerModule lmDataflex(SCLEX_DATAFLEX, ColouriseDataFlexDoc, "dataflex", FoldDataFlexDoc, dataflexWordListDesc);

scintilla/lexers/LexHaskell.cxx

@@ -26,6 +26,7 @@
 #include <ctype.h>
 
 #include <string>
+#include <vector>
 #include <map>
 
 #include "ILexer.h"

scintilla/lexers/LexLaTeX.cxx

@@ -40,6 +40,13 @@ struct latexFoldSave {
 	latexFoldSave(const latexFoldSave &save) : structLev(save.structLev) {
 		for (int i = 0; i < 8; ++i) openBegins[i] = save.openBegins[i];
 	}
+	latexFoldSave &operator=(const latexFoldSave &save) {
+		if (this != &save) {
+			structLev = save.structLev;
+			for (int i = 0; i < 8; ++i) openBegins[i] = save.openBegins[i];
+		}
+		return *this;
+	}
 	int openBegins[8];
 	Sci_Position structLev;
 };

scintilla/lexers/LexMatlab.cxx

@@ -155,7 +155,7 @@ static void ColouriseMatlabOctaveDoc(
 		} else if (sc.state == SCE_MATLAB_KEYWORD) {
 			if (!isalnum(sc.ch) && sc.ch != '_') {
 				char s[100];
-				sc.GetCurrentLowered(s, sizeof(s));
+				sc.GetCurrent(s, sizeof(s));
 
 				if (keywords.InList(s)) {
 					if (strcmp ("end", s) == 0 && allow_end_op) {

scintilla/lexers/LexNim.cxx

@@ -28,7 +28,6 @@
 #include "Accessor.h"
 #include "StyleContext.h"
 #include "CharacterSet.h"
-#include "CharacterCategory.h"
 #include "LexerModule.h"
 #include "OptionSet.h"
 #include "DefaultLexer.h"
@@ -47,7 +46,7 @@ enum NumType {
     FormatError
 };
 
-int GetNumStyle(const int numType) {
+int GetNumStyle(const int numType) noexcept {
     if (numType == NumType::FormatError) {
         return SCE_NIM_NUMERROR;
     }
@@ -55,20 +54,20 @@ int GetNumStyle(const int numType) {
     return SCE_NIM_NUMBER;
 }
 
-bool IsLetter(const int ch) {
+constexpr bool IsLetter(const int ch) noexcept {
     // 97 to 122 || 65 to 90
     return (ch >= 'a' && ch <= 'z') || (ch >= 'A' && ch <= 'Z');
 }
 
-bool IsAWordChar(const int ch) {
+bool IsAWordChar(const int ch) noexcept {
     return ch < 0x80 && (isalnum(ch) || ch == '_' || ch == '.');
 }
 
-int IsNumHex(const StyleContext &sc) {
+int IsNumHex(const StyleContext &sc) noexcept {
     return sc.chNext == 'x' || sc.chNext == 'X';
 }
 
-int IsNumBinary(const StyleContext &sc) {
+int IsNumBinary(const StyleContext &sc) noexcept {
     return sc.chNext == 'b' || sc.chNext == 'B';
 }
 
@@ -76,11 +75,11 @@ int IsNumOctal(const StyleContext &sc) {
     return IsADigit(sc.chNext) || sc.chNext == 'o';
 }
 
-bool IsNewline(const int ch) {
+constexpr bool IsNewline(const int ch) noexcept {
     return (ch == '\n' || ch == '\r');
 }
 
-bool IsFuncName(const char *str) {
+bool IsFuncName(const char *str) noexcept {
     const char *identifiers[] = {
         "proc",
         "func",
@@ -115,7 +114,7 @@ constexpr bool IsStreamComment(const int style) noexcept {
 // Adopted from Accessor.cxx
 int GetIndent(const Sci_Position line, Accessor &styler) {
     Sci_Position startPos = styler.LineStart(line);
-    Sci_Position eolPos = styler.LineStart(line + 1) - 1;
+    const Sci_Position eolPos = styler.LineStart(line + 1) - 1;
 
     char ch = styler[startPos];
     int style = styler.StyleAt(startPos);
@@ -127,7 +126,7 @@ int GetIndent(const Sci_Position line, Accessor &styler) {
     // No fold points inside triple literals
     while ((IsASpaceOrTab(ch) || IsTripleLiteral(style)) && (startPos < eolPos)) {
         if (inPrevPrefix) {
-            char chPrev = styler[posPrev++];
+            const char chPrev = styler[posPrev++];
             if (chPrev != ' ' && chPrev != '\t') {
                 inPrevPrefix = false;
             }
@@ -179,7 +178,7 @@ struct OptionsNim {
 
 static const char *const nimWordListDesc[] = {
     "Keywords",
-    0
+    nullptr
 };
 
 struct OptionSetNim : public OptionSet<OptionsNim> {
@@ -231,11 +230,11 @@ public:
 
     virtual ~LexerNim() { }
 
-    void SCI_METHOD Release() override {
+    void SCI_METHOD Release() noexcept override {
         delete this;
     }
 
-    int SCI_METHOD Version() const override {
+    int SCI_METHOD Version() const noexcept override {
         return lvRelease4;
     }
 
@@ -262,15 +261,15 @@ public:
     void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
     void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
 
-    void * SCI_METHOD PrivateCall(int, void *) override {
+    void * SCI_METHOD PrivateCall(int, void *) noexcept override {
-        return 0;
+        return nullptr;
     }
 
-    int SCI_METHOD LineEndTypesSupported() override {
+    int SCI_METHOD LineEndTypesSupported() noexcept override {
         return SC_LINE_END_TYPE_UNICODE;
     }
 
-    int SCI_METHOD PrimaryStyleFromStyle(int style) override {
+    int SCI_METHOD PrimaryStyleFromStyle(int style) noexcept override {
         return style;
     }
 
@@ -288,7 +287,7 @@ Sci_Position SCI_METHOD LexerNim::PropertySet(const char *key, const char *val)
 }
 
 Sci_Position SCI_METHOD LexerNim::WordListSet(int n, const char *wl) {
-    WordList *wordListN = 0;
+    WordList *wordListN = nullptr;
 
     switch (n) {
         case 0:
@@ -318,7 +317,7 @@ void SCI_METHOD LexerNim::Lex(Sci_PositionU startPos, Sci_Position length,
         initStyle = SCE_NIM_DEFAULT;
     }
 
-    Accessor styler(pAccess, NULL);
+    Accessor styler(pAccess, nullptr);
     StyleContext sc(startPos, length, initStyle, styler);
 
     // Nim supports nested block comments!
@@ -369,8 +368,8 @@ void SCI_METHOD LexerNim::Lex(Sci_PositionU startPos, Sci_Position length,
                         sc.Forward(2);
                     }
                 } else if (sc.ch == '.') {
-                    if (sc.chNext == '.') {
+                    if (IsADigit(sc.chNext)) {
-                        // Pass
+                        sc.Forward();
                     } else if (numType <= NumType::Exponent) {
                         sc.SetState(SCE_NIM_OPERATOR);
                         break;
@@ -391,7 +390,10 @@ void SCI_METHOD LexerNim::Lex(Sci_PositionU startPos, Sci_Position length,
                         }
                     }
                 } else if (sc.ch == '_') {
-                    break;
+                    // Accept only one underscore between digits
+                    if (IsADigit(sc.chNext)) {
+                        sc.Forward();
+                    }
                 } else if (numType == NumType::Decimal) {
                     if (sc.chPrev != '\'' && (sc.ch == 'e' || sc.ch == 'E')) {
                         numType = NumType::Exponent;
@@ -442,7 +444,7 @@ void SCI_METHOD LexerNim::Lex(Sci_PositionU startPos, Sci_Position length,
 
                     if (keywords.InList(s) && !funcNameExists) {
                         // Prevent styling keywords if they are sub-identifiers
-                        Sci_Position segStart = styler.GetStartSegment() - 1;
+                        const Sci_Position segStart = styler.GetStartSegment() - 1;
                         if (segStart < 0 || styler.SafeGetCharAt(segStart, '\0') != '.') {
                             style = SCE_NIM_WORD;
                         }
@@ -592,7 +594,7 @@ void SCI_METHOD LexerNim::Lex(Sci_PositionU startPos, Sci_Position length,
 
         if (sc.state == SCE_NIM_DEFAULT) {
             // Number
-            if (IsADigit(sc.ch) || (IsADigit(sc.chNext) && sc.ch == '.')) {
+            if (IsADigit(sc.ch)) {
                 sc.SetState(SCE_NIM_NUMBER);
 
                 numType = NumType::Decimal;
@@ -624,7 +626,7 @@ void SCI_METHOD LexerNim::Lex(Sci_PositionU startPos, Sci_Position length,
                     sc.SetState(SCE_NIM_STRING);
                 }
 
-                int rawStrStyle = options.highlightRawStrIdent ? IsLetter(sc.ch) :
+                const int rawStrStyle = options.highlightRawStrIdent ? IsLetter(sc.ch) :
                                         (sc.ch == 'r' || sc.ch == 'R');
 
                 if (rawStrStyle) {
@@ -703,15 +705,6 @@ void SCI_METHOD LexerNim::Lex(Sci_PositionU startPos, Sci_Position length,
             // Operators
             else if (strchr("()[]{}:=;-\\/&%$!+<>|^?,.*~@", sc.ch)) {
                 sc.SetState(SCE_NIM_OPERATOR);
-
-                // Ignore decimal coloring in input like: range[0..5]
-                if (sc.Match('.', '.')) {
-                    sc.Forward();
-
-                    if (sc.chNext == '.') {
-                        sc.Forward();
-                    }
-                }
             }
         }
 
@@ -730,7 +723,7 @@ void SCI_METHOD LexerNim::Fold(Sci_PositionU startPos, Sci_Position length, int,
         return;
     }
 
-    Accessor styler(pAccess, NULL);
+    Accessor styler(pAccess, nullptr);
 
     const Sci_Position docLines = styler.GetLine(styler.Length());
     const Sci_Position maxPos = startPos + length;
@@ -778,14 +771,14 @@ void SCI_METHOD LexerNim::Fold(Sci_PositionU startPos, Sci_Position length, int,
         int skipLevel = indentNextLevel;
 
         while (--skipLine > lineCurrent) {
-            int skipLineIndent = IndentAmount(skipLine, styler);
+            const int skipLineIndent = IndentAmount(skipLine, styler);
 
             if (options.foldCompact) {
                 if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > indentNextLevel) {
                     skipLevel = levelBeforeComments;
                 }
 
-                int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
+                const int whiteFlag = skipLineIndent & SC_FOLDLEVELWHITEFLAG;
                 styler.SetLevel(skipLine, skipLevel | whiteFlag);
             } else {
                 if ((skipLineIndent & SC_FOLDLEVELNUMBERMASK) > indentNextLevel &&

scintilla/lexers/LexSQL.cxx

@@ -339,8 +339,8 @@ public :
 	}
 
 	Sci_Position SCI_METHOD WordListSet(int n, const char *wl) override;
-	void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) override;
+	void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
-	void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position lengthDoc, int initStyle, IDocument *pAccess) override;
+	void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
 
 	void * SCI_METHOD PrivateCall(int, void *) override {
 		return 0;
@@ -442,9 +442,8 @@ void SCI_METHOD LexerSQL::Lex(Sci_PositionU startPos, Sci_Position length, int i
 	LexAccessor styler(pAccess);
 	StyleContext sc(startPos, length, initStyle, styler);
 	int styleBeforeDCKeyword = SCE_SQL_DEFAULT;
-	Sci_Position offset = 0;
 
-	for (; sc.More(); sc.Forward(), offset++) {
+	for (; sc.More(); sc.Forward()) {
 		// Determine if the current state should terminate.
 		switch (sc.state) {
 		case SCE_SQL_OPERATOR:
@@ -537,7 +536,7 @@ void SCI_METHOD LexerSQL::Lex(Sci_PositionU startPos, Sci_Position length, int i
 			if (options.sqlBackslashEscapes && sc.ch == '\\') {
 				sc.Forward();
 			} else if (sc.ch == '\'') {
-				if (sc.chNext == '\"') {
+				if (sc.chNext == '\'') {
 					sc.Forward();
 				} else {
 					sc.ForwardSetState(SCE_SQL_DEFAULT);

scintilla/lexers/LexVB.cxx

@@ -53,7 +53,7 @@ static inline bool IsANumberChar(int ch) {
 	// but probably enough in most cases.
 	return (ch < 0x80) &&
 	        (isdigit(ch) || toupper(ch) == 'E' ||
-             ch == '.' || ch == '-' || ch == '+');
+             ch == '.' || ch == '-' || ch == '+' || ch == '_');
 }
 
 static void ColouriseVBDoc(Sci_PositionU startPos, Sci_Position length, int initStyle,
@@ -199,6 +199,10 @@ static void ColouriseVBDoc(Sci_PositionU startPos, Sci_Position length, int init
 				// Octal number
 				sc.SetState(SCE_B_NUMBER);
 				sc.Forward();
+			} else if (sc.ch == '&' && tolower(sc.chNext) == 'b') {
+				// Binary number
+				sc.SetState(SCE_B_NUMBER);
+				sc.Forward();
 			} else if (IsADigit(sc.ch) || (sc.ch == '.' && IsADigit(sc.chNext))) {
 				sc.SetState(SCE_B_NUMBER);
 			} else if (IsAWordStart(sc.ch) || (sc.ch == '[')) {

scintilla/lexers/LexX12.cxx

@@ -0,0 +1,341 @@
+// Scintilla Lexer for X12
+// Written by Iain Clarke, IMCSoft & Inobiz AB.
+// X12 official documentation is behind a paywall, but there's a description of the syntax here:
+// http://www.rawlinsecconsulting.com/x12tutorial/x12syn.html
+// This code is subject to the same license terms as the rest of the scintilla project:
+// The License.txt file describes the conditions under which this software may be distributed.
+//
+
+// Header order must match order in scripts/HeaderOrder.txt
+#include <cstdlib>
+#include <cassert>
+#include <cstring>
+#include <cctype>
+
+#include <vector>
+#include <algorithm>
+
+#include "ILexer.h"
+#include "Scintilla.h"
+#include "SciLexer.h"
+#include "LexerModule.h"
+#include "DefaultLexer.h"
+
+using namespace Scintilla;
+
+class LexerX12 : public DefaultLexer
+{
+public:
+	LexerX12();
+	virtual ~LexerX12() {} // virtual destructor, as we inherit from ILexer
+
+	static ILexer4 *Factory() {
+		return new LexerX12;
+	}
+
+	int SCI_METHOD Version() const override
+	{
+		return lvRelease4;
+	}
+	void SCI_METHOD Release() override
+	{
+		delete this;
+	}
+
+	const char * SCI_METHOD PropertyNames() override
+	{
+		return "fold";
+	}
+	int SCI_METHOD PropertyType(const char *) override
+	{
+		return SC_TYPE_BOOLEAN; // Only one property!
+	}
+	const char * SCI_METHOD DescribeProperty(const char *name) override
+	{
+		if (!strcmp(name, "fold"))
+			return "Whether to apply folding to document or not";
+		return NULL;
+	}
+
+	Sci_Position SCI_METHOD PropertySet(const char *key, const char *val) override
+	{
+		if (!strcmp(key, "fold"))
+		{
+			m_bFold = strcmp(val, "0") ? true : false;
+			return 0;
+		}
+		return -1;
+	}
+	const char * SCI_METHOD DescribeWordListSets() override
+	{
+		return NULL;
+	}
+	Sci_Position SCI_METHOD WordListSet(int, const char *) override
+	{
+		return -1;
+	}
+	void SCI_METHOD Lex(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
+	void SCI_METHOD Fold(Sci_PositionU startPos, Sci_Position length, int initStyle, IDocument *pAccess) override;
+	void * SCI_METHOD PrivateCall(int, void *) override
+	{
+		return NULL;
+	}
+
+protected:
+	struct Terminator
+	{
+		int Style = SCE_X12_BAD;
+		Sci_PositionU pos = 0;
+		Sci_PositionU length = 0;
+		int FoldChange = 0;
+	};
+	Terminator InitialiseFromISA(IDocument *pAccess);
+	Sci_PositionU FindPreviousSegmentStart(IDocument *pAccess, Sci_Position startPos) const;
+	Terminator DetectSegmentHeader(IDocument *pAccess, Sci_PositionU pos) const;
+	Terminator FindNextTerminator(IDocument *pAccess, Sci_PositionU pos, bool bJustSegmentTerminator = false) const;
+
+	bool m_bFold;
+	char m_chSubElement;
+	char m_chElement;
+	char m_chSegment[3]; // might be CRLF
+};
+
+LexerModule lmX12(SCLEX_X12, LexerX12::Factory, "x12");
+
+///////////////////////////////////////////////////////////////////////////////
+
+
+
+///////////////////////////////////////////////////////////////////////////////
+
+LexerX12::LexerX12()
+{
+	m_bFold = false;
+	m_chSegment[0] = m_chSegment[1] = m_chSegment[2] = m_chElement = m_chSubElement = 0;
+}
+
+void LexerX12::Lex(Sci_PositionU startPos, Sci_Position length, int, IDocument *pAccess)
+{
+	Sci_PositionU posFinish = startPos + length;
+
+	Terminator T = InitialiseFromISA(pAccess);
+
+	if (T.Style == SCE_X12_BAD)
+	{
+		if (T.pos < startPos)
+			T.pos = startPos; // we may be colouring in batches.
+		pAccess->StartStyling(startPos);
+		pAccess->SetStyleFor(T.pos - startPos, SCE_X12_ENVELOPE);
+		pAccess->SetStyleFor(posFinish - T.pos, SCE_X12_BAD);
+		return;
+	}
+
+	// Look backwards for a segment start or a document beginning
+	Sci_PositionU posCurrent = FindPreviousSegmentStart (pAccess, startPos);
+
+	// Style buffer, so we're not issuing loads of notifications
+	pAccess->StartStyling(posCurrent);
+
+	while (posCurrent < posFinish)
+	{
+		// Look for first element marker, so we can denote segment
+		T = DetectSegmentHeader(pAccess, posCurrent);
+		if (T.Style == SCE_X12_BAD)
+			break;
+
+		pAccess->SetStyleFor(T.pos - posCurrent, T.Style);
+		pAccess->SetStyleFor(T.length, SCE_X12_SEP_ELEMENT);
+		posCurrent = T.pos + T.length;
+
+		while (T.Style != SCE_X12_BAD && T.Style != SCE_X12_SEGMENTEND) // Break on bad or segment ending
+		{
+			T = FindNextTerminator(pAccess, posCurrent);
+			if (T.Style == SCE_X12_BAD)
+				break;
+
+			int Style = T.Style;
+			if (T.Style == SCE_X12_SEGMENTEND && m_chSegment[0] == '\r') // don't style cr/crlf
+				Style = SCE_X12_DEFAULT;
+
+			pAccess->SetStyleFor(T.pos - posCurrent, SCE_X12_DEFAULT);
+			pAccess->SetStyleFor(T.length, Style);
+			posCurrent = T.pos + T.length;
+		}
+		if (T.Style == SCE_X12_BAD)
+			break;
+	}
+
+	pAccess->SetStyleFor(posFinish - posCurrent, SCE_X12_BAD);
+}
+
+void LexerX12::Fold(Sci_PositionU startPos, Sci_Position length, int, IDocument *pAccess)
+{
+	if (!m_bFold)
+		return;
+
+	// Are we even foldable?
+	if (m_chSegment[0] != '\r' && m_chSegment[0] != '\n') // check for cr,lf,cr+lf.
+		return;
+
+	Sci_PositionU posFinish = startPos + length;
+
+	// Look backwards for a segment start or a document beginning
+	startPos = FindPreviousSegmentStart(pAccess, startPos);
+	Terminator T;
+
+	Sci_PositionU currLine = pAccess->LineFromPosition(startPos);
+	int levelCurrentStyle = SC_FOLDLEVELBASE;
+	if (currLine > 0)
+		levelCurrentStyle = pAccess->GetLevel(currLine - 1); // bottom 12 bits are level
+	int indentCurrent = levelCurrentStyle & (SC_FOLDLEVELBASE - 1);
+
+	while (startPos < posFinish)
+	{
+		T = DetectSegmentHeader(pAccess, startPos);
+		int indentNext = indentCurrent + T.FoldChange;
+		if (indentNext < 0)
+			indentNext = 0;
+
+		levelCurrentStyle = (T.FoldChange > 0) ? (SC_FOLDLEVELBASE | SC_FOLDLEVELHEADERFLAG) : SC_FOLDLEVELBASE;
+
+		currLine = pAccess->LineFromPosition(startPos);
+		pAccess->SetLevel(currLine, levelCurrentStyle | indentCurrent);
+
+		T = FindNextTerminator(pAccess, startPos, true);
+		startPos = T.pos + T.length;
+		indentCurrent = indentNext;
+	}
+}
+
+LexerX12::Terminator LexerX12::InitialiseFromISA(IDocument *pAccess)
+{
+	Sci_Position length = pAccess->Length();
+	char c;
+	if (length <= 106)
+		return { SCE_X12_BAD, 0 };
+
+	pAccess->GetCharRange(&m_chElement, 3, 1);
+	pAccess->GetCharRange(&m_chSubElement, 104, 1);
+	pAccess->GetCharRange(m_chSegment, 105, 1);
+	if (m_chSegment[0] == '\r') // are we CRLF?
+	{
+		pAccess->GetCharRange(&c, 106, 1);
+		if (c == '\n')
+			m_chSegment[1] = c;
+	}
+
+	// Validate we have an element separator, and it's not silly!
+	if (m_chElement == '\0' || m_chElement == '\n' || m_chElement == '\r')
+		return { SCE_X12_BAD, 3 };
+
+	// Validate we have an element separator, and it's not silly!
+	if (m_chSubElement == '\0' || m_chSubElement == '\n' || m_chSubElement == '\r')
+		return { SCE_X12_BAD, 103 };
+
+	if (m_chElement == m_chSubElement)
+		return { SCE_X12_BAD, 104 };
+	if (m_chElement == m_chSegment[0])
+		return { SCE_X12_BAD, 105 };
+	if (m_chSubElement == m_chSegment[0])
+		return { SCE_X12_BAD, 104 };
+
+	// Check we have element markers at all the right places! ISA element has fixed entries.
+	std::vector<Sci_PositionU> ElementMarkers = { 3, 6, 17, 20, 31, 34, 50, 53, 69, 76, 81, 83, 89, 99, 101, 103  };
+	for (auto i : ElementMarkers)
+	{
+		pAccess->GetCharRange(&c, i, 1);
+		if (c != m_chElement)
+			return { SCE_X12_BAD, i };
+	}
+	// Check we have no element markers anywhere else!
+	for (Sci_PositionU i = 0; i < 105; i++)
+	{
+		if (std::find(ElementMarkers.begin(), ElementMarkers.end(), i) != ElementMarkers.end())
+			continue;
+
+		pAccess->GetCharRange(&c, i, 1);
+		if (c == m_chElement)
+			return { SCE_X12_BAD, i };
+	}
+
+	return { SCE_X12_ENVELOPE };
+}
+
+Sci_PositionU LexerX12::FindPreviousSegmentStart(IDocument *pAccess, Sci_Position startPos) const
+{
+	char c;
+
+	for ( ; startPos > 0; startPos--)
+	{
+		pAccess->GetCharRange(&c, startPos, 1);
+		if (c != m_chSegment[0])
+			continue;
+		// we've matched one - if this is not crlf we're done.
+		if (!m_chSegment[1])
+			return startPos + 1;
+		pAccess->GetCharRange(&c, startPos+1, 1);
+		if (c == m_chSegment[1])
+			return startPos + 2;
+	}
+	// We didn't find a ', so just go with the beginning
+	return 0;
+}
+
+LexerX12::Terminator LexerX12::DetectSegmentHeader(IDocument *pAccess, Sci_PositionU pos) const
+{
+	Sci_PositionU posStart = pos;
+	Sci_Position Length = pAccess->Length();
+	char Buf[6] = { 0 };
+	while (pos - posStart < 5 && pos < (Sci_PositionU)Length)
+	{
+		pAccess->GetCharRange(Buf + pos - posStart, pos, 1);
+		if (Buf [pos - posStart] != m_chElement) // more?
+		{
+			pos++;
+			continue;
+		}
+		if (strcmp(Buf, "ISA*") == 0)
+			return { SCE_X12_ENVELOPE, pos, 1, +1 };
+		if (strcmp(Buf, "IEA*") == 0)
+			return { SCE_X12_ENVELOPE, pos, 1, -1 };
+		if (strcmp(Buf, "GS*") == 0)
+			return { SCE_X12_FUNCTIONGROUP, pos, 1, +1 };
+		if (strcmp(Buf, "GE*") == 0)
+			return { SCE_X12_FUNCTIONGROUP, pos, 1, -1 };
+		if (strcmp(Buf, "ST*") == 0)
+			return { SCE_X12_TRANSACTIONSET, pos, 1, +1 };
+		if (strcmp(Buf, "SE*") == 0)
+			return { SCE_X12_TRANSACTIONSET, pos, 1, -1 };
+		return { SCE_X12_SEGMENTHEADER, pos, 1, 0 };
+	}
+	return { SCE_X12_BAD, pos, 0, 0 };
+}
+
+LexerX12::Terminator LexerX12::FindNextTerminator(IDocument *pAccess, Sci_PositionU pos, bool bJustSegmentTerminator) const
+{
+	char c;
+	Sci_Position Length = pAccess->Length();
+
+	while (pos < (Sci_PositionU)Length)
+	{
+		pAccess->GetCharRange(&c, pos, 1);
+		if (!bJustSegmentTerminator && c == m_chElement)
+			return { SCE_X12_SEP_ELEMENT, pos, 1 };
+		else if (!bJustSegmentTerminator && c == m_chSubElement)
+			return { SCE_X12_SEP_SUBELEMENT, pos, 1 };
+		else if (c == m_chSegment[0])
+		{
+			if (!m_chSegment[1])
+				return { SCE_X12_SEGMENTEND, pos, 1 };
+			pos++;
+			if (pos >= (Sci_PositionU)Length)
+				break;
+			pAccess->GetCharRange(&c, pos, 1);
+			if (c == m_chSegment[1])
+				return { SCE_X12_SEGMENTEND, pos-1, 2 };
+		}
+		pos++;
+	}
+
+	return { SCE_X12_BAD, pos };
+}

scintilla/lexlib/CharacterCategory.cxx

@@ -7,6 +7,7 @@
 // Copyright 2013 by Neil Hodgson <neilh@scintilla.org>
 // The License.txt file describes the conditions under which this software may be distributed.
 
+#include <vector>
 #include <algorithm>
 #include <iterator>
 
@@ -3790,6 +3791,7 @@ const int catRanges[] = {
 33554397,
 33554460,
 35651549,
+35651613,
 //--Autogenerated -- end of section automatically generated
 };
 
@@ -3963,4 +3965,33 @@ bool IsXidContinue(int character) {
 	}
 }
 
+CharacterCategoryMap::CharacterCategoryMap() noexcept {
+	Optimize(256);
+}
+
+int CharacterCategoryMap::Size() const noexcept {
+	return static_cast<int>(dense.size());
+}
+
+void CharacterCategoryMap::Optimize(int countCharacters) {
+	const int characters = std::clamp(countCharacters, 256, maxUnicode + 1);
+	dense.resize(characters);
+
+	int end = 0;
+	int index = 0;
+	int current = catRanges[index];
+	++index;
+	do {
+		const int next = catRanges[index];
+		const unsigned char category = current & maskCategory;
+		current >>= 5;
+		end = std::min(characters, next >> 5);
+		while (current < end) {
+			dense[current++] = category;
+		}
+		current = next;
+		++index;
+	} while (characters > end);
+}
+
 }

scintilla/lexlib/CharacterCategory.h

@@ -28,6 +28,23 @@ bool IsIdContinue(int character);
 bool IsXidStart(int character);
 bool IsXidContinue(int character);
 
+class CharacterCategoryMap {
+private:
+	std::vector<unsigned char> dense;
+public:
+	CharacterCategoryMap() noexcept;
+	CharacterCategory CategoryFor(int character) const {
+		if (static_cast<size_t>(character) < dense.size()) {
+			return static_cast<CharacterCategory>(dense[character]);
+		} else {
+			// binary search through ranges
+			return CategoriseCharacter(character);
+		}
+	}
+	int Size() const noexcept;
+	void Optimize(int countCharacters);
+};
+
 }
 
 #endif

scintilla/lexlib/CharacterSet.h

@@ -82,10 +82,9 @@ public:
 	}
 	void AddString(const char *setToAdd) {
 		for (const char *cp=setToAdd; *cp; cp++) {
-			int val = static_cast<unsigned char>(*cp);
+			const unsigned char uch = *cp;
-			assert(val >= 0);
+			assert(uch < size);
-			assert(val < size);
+			bset[uch] = true;
-			bset[val] = true;
 		}
 	}
 	bool Contains(int val) const {
@@ -93,6 +92,11 @@ public:
 		if (val < 0) return false;
 		return (val < size) ? bset[val] : valueAfter;
 	}
+	bool Contains(char ch) const {
+		// Overload char as char may be signed
+		const unsigned char uch = ch;
+		return Contains(uch);
+	}
 };
 
 // Functions for classifying characters
@@ -131,6 +135,10 @@ inline bool IsUpperCase(int ch) {
 	return (ch >= 'A') && (ch <= 'Z');
 }
 
+inline bool IsUpperOrLowerCase(int ch) {
+	return IsUpperCase(ch) || IsLowerCase(ch);
+}
+
 inline bool IsAlphaNumeric(int ch) {
 	return
 		((ch >= '0') && (ch <= '9')) ||

scintilla/lexlib/SubStyles.h

@@ -61,7 +61,19 @@ public:
 		return (style >= firstStyle) && (style < (firstStyle + lenStyles));
 	}
 
+	void RemoveStyle(int style) {
+		std::map<std::string, int>::iterator it = wordToStyle.begin();
+		while (it != wordToStyle.end()) {
+			if (it->second == style) {
+				it = wordToStyle.erase(it);
+			} else {
+				++it;
+			}
+		}
+	}
+
 	void SetIdentifiers(int style, const char *identifiers) {
+		RemoveStyle(style);
 		while (*identifiers) {
 			const char *cpSpace = identifiers;
 			while (*cpSpace && !(*cpSpace == ' ' || *cpSpace == '\t' || *cpSpace == '\r' || *cpSpace == '\n'))

scintilla/lexlib/WordList.h

@@ -31,6 +31,7 @@ public:
 	bool InListAbbreviated(const char *s, const char marker) const;
 	bool InListAbridged(const char *s, const char marker) const;
 	const char *WordAt(int n) const;
+	
 	void SetWordAt(int n, const char *word2Set) {
 		words[n] = (char *)word2Set;
 	};

scintilla/qt/ScintillaEdit/ScintillaDocument.cpp

@@ -16,6 +16,7 @@
 #include "ILexer.h"
 #include "Scintilla.h"
 
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"

scintilla/qt/ScintillaEdit/ScintillaEdit.h.template

@@ -65,6 +65,9 @@ public:
 
 #if defined(__GNUC__)
 #pragma GCC diagnostic ignored "-Wmissing-field-initializers"
+#if !defined(__clang__) && (__GNUC__ >= 8)
+#pragma GCC diagnostic ignored "-Wcast-function-type"
+#endif
 #endif
 
 #endif /* SCINTILLAEDIT_H */

scintilla/qt/ScintillaEdit/ScintillaEdit.pro

@@ -12,7 +12,7 @@ TEMPLATE = lib
 CONFIG += lib_bundle
 CONFIG += c++1z
 
-VERSION = 4.1.4
+VERSION = 4.2.0
 
 SOURCES += \
     ScintillaEdit.cpp \
@@ -22,6 +22,7 @@ SOURCES += \
     ../ScintillaEditBase/ScintillaEditBase.cpp \
     ../../src/XPM.cxx \
     ../../src/ViewStyle.cxx \
+    ../../src/UniqueString.cxx \
     ../../src/UniConversion.cxx \
     ../../src/Style.cxx \
     ../../src/Selection.cxx \

scintilla/qt/ScintillaEdit/WidgetGen.py

@@ -39,6 +39,8 @@ def normalisedName(s, options, role=None):
 
 typeAliases = {
 	"position": "int",
+	"line": "int",
+	"pointer": "int",
 	"colour": "int",
 	"keymod": "int",
 	"string": "const char *",
@@ -49,21 +51,26 @@ typeAliases = {
 def cppAlias(s):
 	if s in typeAliases:
 		return typeAliases[s]
+	elif Face.IsEnumeration(s):
+		return "int"
 	else:
 		return s
 
-understoodTypes = ["", "void", "int", "bool", "position",
+understoodTypes = ["", "void", "int", "bool", "position", "line", "pointer",
 	"colour", "keymod", "string", "stringresult", "cells"]
 
+def understoodType(t):
+	return t in understoodTypes or Face.IsEnumeration(t)
+
 def checkTypes(name, v):
 	understandAllTypes = True
-	if v["ReturnType"] not in understoodTypes:
+	if not understoodType(v["ReturnType"]):
 		#~ print("Do not understand", v["ReturnType"], "for", name)
 		understandAllTypes = False
-	if v["Param1Type"] not in understoodTypes:
+	if not understoodType(v["Param1Type"]):
 		#~ print("Do not understand", v["Param1Type"], "for", name)
 		understandAllTypes = False
-	if v["Param2Type"] not in understoodTypes:
+	if not understoodType(v["Param2Type"]):
 		#~ print("Do not understand", v["Param2Type"], "for", name)
 		understandAllTypes = False
 	return understandAllTypes

scintilla/qt/ScintillaEditBase/ScintillaEditBase.cpp

@@ -23,11 +23,10 @@
 
 #define INDIC_INPUTMETHOD 24
 
-#define MAXLENINPUTIME 200
+#define SC_INDICATOR_INPUT INDICATOR_IME
-#define SC_INDICATOR_INPUT INDIC_IME
+#define SC_INDICATOR_TARGET INDICATOR_IME+1
-#define SC_INDICATOR_TARGET INDIC_IME+1
+#define SC_INDICATOR_CONVERTED INDICATOR_IME+2
-#define SC_INDICATOR_CONVERTED INDIC_IME+2
+#define SC_INDICATOR_UNKNOWN INDICATOR_IME_MAX
-#define SC_INDICATOR_UNKNOWN INDIC_IME_MAX
 
 // Q_WS_MAC and Q_WS_X11 aren't defined in Qt5
 #if QT_VERSION >= QT_VERSION_CHECK(5, 0, 0)
@@ -259,7 +258,7 @@ void ScintillaEditBase::keyPressEvent(QKeyEvent *event)
 		QString text = event->text();
 		if (input && !text.isEmpty() && text[0].isPrint()) {
 			QByteArray utext = sqt->BytesForDocument(text);
-			sqt->AddCharUTF(utext.data(), utext.size());
+			sqt->InsertCharacter(std::string_view(utext.data(), utext.size()), EditModel::CharacterSource::directInput);
 		} else {
 			event->ignore();
 		}
@@ -450,9 +449,9 @@ void ScintillaEditBase::DrawImeIndicator(int indicator, int len)
 {
 	// Emulate the visual style of IME characters with indicators.
 	// Draw an indicator on the character before caret by the character bytes of len
-	// so it should be called after AddCharUTF().
+	// so it should be called after InsertCharacter().
 	// It does not affect caret positions.
-	if (indicator < 8 || indicator > INDIC_MAX) {
+	if (indicator < 8 || indicator > INDICATOR_MAX) {
 		return;
 	}
 	sqt->pdoc->DecorationSetCurrentIndicator(indicator);
@@ -548,16 +547,15 @@ void ScintillaEditBase::inputMethodEvent(QInputMethodEvent *event)
 			const unsigned int ucWidth = commitStr.at(i).isHighSurrogate() ? 2 : 1;
 			const QString oneCharUTF16 = commitStr.mid(i, ucWidth);
 			const QByteArray oneChar = sqt->BytesForDocument(oneCharUTF16);
-			const int oneCharLen = oneChar.length();
 
-			sqt->AddCharUTF(oneChar.data(), oneCharLen);
+			sqt->InsertCharacter(std::string_view(oneChar.data(), oneChar.length()), EditModel::CharacterSource::directInput);
 			i += ucWidth;
 		}
 
 	} else if (!event->preeditString().isEmpty()) {
 		const QString preeditStr = event->preeditString();
 		const unsigned int preeditStrLen = preeditStr.length();
-		if ((preeditStrLen == 0) || (preeditStrLen > MAXLENINPUTIME)) {
+		if (preeditStrLen == 0) {
 			sqt->ShowCaretAtCurrentPosition();
 			return;
 		}
@@ -568,20 +566,17 @@ void ScintillaEditBase::inputMethodEvent(QInputMethodEvent *event)
 
 		std::vector<int> imeIndicator = MapImeIndicators(event);
 
-		const bool recording = sqt->recordingMacro;
-		sqt->recordingMacro = false;
 		for (unsigned int i = 0; i < preeditStrLen;) {
 			const unsigned int ucWidth = preeditStr.at(i).isHighSurrogate() ? 2 : 1;
 			const QString oneCharUTF16 = preeditStr.mid(i, ucWidth);
 			const QByteArray oneChar = sqt->BytesForDocument(oneCharUTF16);
 			const int oneCharLen = oneChar.length();
 
-			sqt->AddCharUTF(oneChar.data(), oneCharLen);
+			sqt->InsertCharacter(std::string_view(oneChar.data(), oneCharLen), EditModel::CharacterSource::tentativeInput);
 
 			DrawImeIndicator(imeIndicator[i], oneCharLen);
 			i += ucWidth;
 		}
-		sqt->recordingMacro = recording;
 
 		// Move IME carets.
 		int imeCaretPos = GetImeCaretPos(event);

scintilla/qt/ScintillaEditBase/ScintillaEditBase.pro

@@ -12,7 +12,7 @@ TEMPLATE = lib
 CONFIG += lib_bundle
 CONFIG += c++1z
 
-VERSION = 4.1.4
+VERSION = 4.2.0
 
 SOURCES += \
     PlatQt.cpp \
@@ -20,6 +20,7 @@ SOURCES += \
     ScintillaEditBase.cpp \
     ../../src/XPM.cxx \
     ../../src/ViewStyle.cxx \
+    ../../src/UniqueString.cxx \
     ../../src/UniConversion.cxx \
     ../../src/Style.cxx \
     ../../src/Selection.cxx \

scintilla/qt/ScintillaEditBase/ScintillaQt.h

@@ -31,6 +31,7 @@
 #include "Platform.h"
 #include "ILoader.h"
 #include "ILexer.h"
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"

scintilla/scripts/CheckMentioned.py

@@ -0,0 +1,223 @@
+#!/usr/bin/env python3
+# CheckMentioned.py
+# Find all the symbols in scintilla/include/Scintilla.h and check if they
+# are mentioned in scintilla/doc/ScintillaDoc.html.
+# Requires Python 2.7 or later
+
+import re, string, sys
+
+srcRoot = "../.."
+
+sys.path.append(srcRoot + "/scintilla/scripts")
+
+import Face
+import ScintillaData
+
+uninteresting = {
+	"SCINTILLA_H", "SCI_START", "SCI_LEXER_START", "SCI_OPTIONAL_START",
+	# These archaic names are #defined to the Sci_ prefixed modern equivalents.
+	# They are not documented so they are not used in new code.
+	"CharacterRange", "TextRange", "TextToFind", "RangeToFormat", "NotifyHeader",
+}
+
+incFileName = srcRoot + "/scintilla/include/Scintilla.h"
+docFileName = srcRoot + "/scintilla/doc/ScintillaDoc.html"
+try:	# Old Python
+	identCharacters = "_" + string.letters + string.digits
+except AttributeError:	# Python 3.x
+	identCharacters = "_" + string.ascii_letters + string.digits
+
+# Convert all punctuation characters except '_' into spaces.
+def depunctuate(s):
+	d = ""
+	for ch in s:
+		if ch in identCharacters:
+			d = d + ch
+		else:
+			d = d + " "
+	return d
+
+symbols = {}
+with open(incFileName, "rt") as incFile:
+	for line in incFile.readlines():
+		if line.startswith("#define"):
+			identifier = line.split()[1]
+			symbols[identifier] = 0
+
+with open(docFileName, "rt") as docFile:
+	for line in docFile.readlines():
+		for word in depunctuate(line).split():
+			if word in symbols.keys():
+				symbols[word] = 1
+
+def convertIFaceTypeToC(t):
+	if t == "keymod":
+		return "int "
+	elif t == "string":
+		return "const char *"
+	elif t == "stringresult":
+		return "char *"
+	elif t == "cells":
+		return "cell *"
+	elif t == "textrange":
+		return "Sci_TextRange *"
+	elif t == "findtext":
+		return "Sci_TextToFind *"
+	elif t == "formatrange":
+		return "Sci_RangeToFormat *"
+	elif Face.IsEnumeration(t):
+		return "int "
+	return t + " "
+
+def makeParm(t, n, v):
+	return (convertIFaceTypeToC(t) + n).rstrip()
+
+def makeRet(params):
+	retType = params["ReturnType"]
+	if retType in ["void", "string", "stringresult"]:
+		retType = ""
+	if retType:
+		retType = " → " + retType
+
+	return retType
+
+def makeSig(params):
+	p1 = makeParm(params["Param1Type"], params["Param1Name"], params["Param1Value"])
+	p2 = makeParm(params["Param2Type"], params["Param2Name"], params["Param2Value"])
+
+	retType = params["ReturnType"]
+	if retType in ["void", "string", "stringresult"]:
+		retType = ""
+	elif Face.IsEnumeration(retType):
+		retType = "int"
+	if retType:
+		retType = " → " + retType
+
+	if p1 == "" and p2 == "":
+		return retType
+
+	ret = ""
+	if p1 == "":
+		p1 = "<unused>"
+	joiner = ""
+	if p2 != "":
+		joiner = ", "
+	return "(" + p1 + joiner + p2 + ")" + retType
+
+pathIface = srcRoot + "/scintilla/include/Scintilla.iface"
+
+def retrieveFeatures():
+	sci = ScintillaData.ScintillaData(srcRoot + "/scintilla/")
+	face = Face.Face()
+	face.ReadFromFile(pathIface)
+	sciToFeature = {}
+	sccToValue = { "true":"1", "false":"0", "EN_SETFOCUS":"256", "EN_KILLFOCUS":"512"}
+	for name in face.order:
+		v = face.features[name]
+		if v["FeatureType"] in ["fun", "get", "set"]:
+			featureDefineName = "SCI_" + name.upper()
+			sciToFeature[featureDefineName] = name
+		elif v["FeatureType"] in ["val"]:
+			featureDefineName = name.upper()
+			sccToValue[featureDefineName] = v["Value"]
+		elif v["FeatureType"] in ["evt"]:
+			featureDefineName = "SCN_" + name.upper()
+			sccToValue[featureDefineName] = v["Value"]
+	return (face, sciToFeature, sccToValue)
+
+def flattenSpaces(s):
+	return s.replace("\n", " ").replace("  ", " ").replace("  ", " ").replace("  ", " ").strip()
+
+def printCtag(ident, path):
+	print(ident.strip() + "\t" + path + "\t" + "/^" + ident + "$/")
+
+showCTags = True
+
+def checkDocumentation():
+	with open(docFileName, "rt") as docFile:
+		docs = docFile.read()
+
+	face, sciToFeature, sccToValue = retrieveFeatures()
+
+	headers = {}
+	definitions = {}
+
+	# Examine header sections which point to definitions
+	#<a class="message" href="#SCI_SETLAYOUTCACHE">SCI_SETLAYOUTCACHE(int cacheMode)</a><br />
+	dirPattern = re.compile(r'<a class="message" href="#([A-Z0-9_]+)">([A-Z][A-Za-z0-9_() *&;,\n]+)</a>')
+	firstWord = re.compile(r'[A-Z0-9_]+')
+	for api, sig in re.findall(dirPattern, docs):
+		sigApi = re.split('\W+', sig)[0]
+		sigFlat = flattenSpaces(sig)
+		sigFlat = sigFlat.replace('alpha ', 'int ')
+		sigFlat = sigFlat.replace("document *", "int ")
+		sigFlat = sigFlat.rstrip()
+		if '(' in sigFlat or api.startswith("SCI_"):
+			name = sciToFeature[api]
+			sigFromFace = api + makeSig(face.features[name])
+			if sigFlat != sigFromFace:
+				print(sigFlat, "|", sigFromFace)
+				if showCTags:
+					printCtag(api, docFileName)
+				#~ printCtag(" " + name, pathIface)
+		if api != sigApi:
+			print(sigApi, ";;", sig, ";;", api)
+		headers[api] = 1
+	# Warns for most keyboard commands so not enabled
+	#~ for api in sorted(sciToFeature.keys()):
+		#~ if api not in headers:
+			#~ print("No header for ", api)
+
+	# Examine  definitions
+	#<b id="SCI_SETLAYOUTCACHE">SCI_SETLAYOUTCACHE(int cacheMode)</b>
+	defPattern = re.compile(r'<b id="([A-Z_0-9]+)">([A-Z][A-Za-z0-9_() *#\"=<>/&;,\n-]+?)</b>')
+	for api, sig in re.findall(defPattern, docs):
+		sigFlat = flattenSpaces(sig)
+		if '<a' in sigFlat	:	# Remove anchors
+			sigFlat = re.sub('<a.*>(.+)</a>', '\\1', sigFlat)
+		sigFlat = sigFlat.replace('alpha ', 'int ')
+		sigFlat = sigFlat.replace("document *", "int ")
+		sigFlat = sigFlat.replace(' NUL-terminated', '')
+		sigFlat = sigFlat.rstrip()
+		#~ sigFlat = sigFlat.replace(' NUL-terminated', '')
+		sigApi = re.split('\W+', sigFlat)[0]
+		#~ print(sigFlat, ";;", sig, ";;", api)
+		if '(' in sigFlat or api.startswith("SCI_"):
+			try:
+				name = sciToFeature[api]
+				sigFromFace = api + makeSig(face.features[name])
+				if sigFlat != sigFromFace:
+					print(sigFlat, "|", sigFromFace)
+					if showCTags:
+						printCtag('="' + api, docFileName)
+					#~ printCtag(" " + name, pathIface)
+			except KeyError:
+				pass		# Feature removed but still has documentation
+		if api != sigApi:
+			print(sigApi, ";;", sig, ";;", api)
+		definitions[api] = 1
+	# Warns for most keyboard commands so not enabled
+	#~ for api in sorted(sciToFeature.keys()):
+		#~ if api not in definitions:
+			#~ print("No definition for ", api)
+
+	outName = docFileName.replace("Doc", "Dox")
+	with open(outName, "wt") as docFile:
+		docFile.write(docs)
+
+	# Examine  constant definitions
+	#<code>SC_CARETSTICKY_WHITESPACE</code> (2)
+	constPattern = re.compile(r'<code>(\w+)</code> *\((\w+)\)')
+	for name, val in re.findall(constPattern, docs):
+		try:
+			valOfName = sccToValue[name]
+			if val != valOfName:
+				print(val, "<-", name, ";;", valOfName)
+		except KeyError:
+			print("***", val, "<-", name)
+
+for identifier in sorted(symbols.keys()):
+	if not symbols[identifier] and identifier not in uninteresting:
+		print(identifier)
+
+checkDocumentation()

scintilla/scripts/Dependencies.py

@@ -0,0 +1,153 @@
+#!/usr/bin/env python
+# Dependencies.py - discover, read, and write dependencies file for make.
+# The format like the output from "g++ -MM" which produces a
+# list of header (.h) files used by source files (.cxx).
+# As a module, provides
+#	FindPathToHeader(header, includePath) -> path
+#	FindHeadersInFile(filePath) -> [headers]
+#	FindHeadersInFileRecursive(filePath, includePath, renames) -> [paths]
+#	FindDependencies(sourceGlobs, includePath, objExt, startDirectory, renames) -> [dependencies]
+#	ExtractDependencies(input) -> [dependencies]
+#	TextFromDependencies(dependencies)
+#	WriteDependencies(output, dependencies)
+#	UpdateDependencies(filepath, dependencies)
+#	PathStem(p) -> stem
+#	InsertSynonym(dependencies, current, additional) -> [dependencies]
+# If run as a script reads from stdin and writes to stdout.
+# Only tested with ASCII file names.
+# Copyright 2019 by Neil Hodgson <neilh@scintilla.org>
+# The License.txt file describes the conditions under which this software may be distributed.
+# Requires Python 2.7 or later
+
+import codecs, glob, os, sys
+
+if __name__ == "__main__":
+	import FileGenerator
+else:
+	from . import FileGenerator
+
+continuationLineEnd = " \\"
+
+def FindPathToHeader(header, includePath):
+	for incDir in includePath:
+		relPath = os.path.join(incDir, header)
+		if os.path.exists(relPath):
+			return relPath
+	return ""
+
+fhifCache = {}	# Remember the includes in each file. ~5x speed up.
+def FindHeadersInFile(filePath):
+	if filePath not in fhifCache:
+		headers = []
+		with codecs.open(filePath, "r", "utf-8") as f:
+			for line in f:
+				if line.strip().startswith("#include"):
+					parts = line.split()
+					if len(parts) > 1:
+						header = parts[1]
+						if header[0] != '<':	# No system headers
+							headers.append(header.strip('"'))
+		fhifCache[filePath] = headers
+	return fhifCache[filePath]
+
+def FindHeadersInFileRecursive(filePath, includePath, renames):
+	headerPaths = []
+	for header in FindHeadersInFile(filePath):
+		if header in renames:
+			header = renames[header]
+		relPath = FindPathToHeader(header, includePath)
+		if relPath and relPath not in headerPaths:
+				headerPaths.append(relPath)
+				subHeaders = FindHeadersInFileRecursive(relPath, includePath, renames)
+				headerPaths.extend(sh for sh in subHeaders if sh not in headerPaths)
+	return headerPaths
+
+def RemoveStart(relPath, start):
+	if relPath.startswith(start):
+		return relPath[len(start):]
+	return relPath
+
+def ciKey(f):
+	return f.lower()
+
+def FindDependencies(sourceGlobs, includePath, objExt, startDirectory, renames={}):
+	deps = []
+	for sourceGlob in sourceGlobs:
+		sourceFiles = glob.glob(sourceGlob)
+		# Sorting the files minimizes deltas as order returned by OS may be arbitrary
+		sourceFiles.sort(key=ciKey)
+		for sourceName in sourceFiles:
+			objName = os.path.splitext(os.path.basename(sourceName))[0]+objExt
+			headerPaths = FindHeadersInFileRecursive(sourceName, includePath, renames)
+			depsForSource = [sourceName] + headerPaths
+			depsToAppend = [RemoveStart(fn.replace("\\", "/"), startDirectory) for
+				fn in depsForSource]
+			deps.append([objName, depsToAppend])
+	return deps
+
+def PathStem(p):
+	""" Return the stem of a filename: "CallTip.o" -> "CallTip" """
+	return os.path.splitext(os.path.basename(p))[0]
+
+def InsertSynonym(dependencies, current, additional):
+	""" Insert a copy of one object file with dependencies under a different name.
+	Used when one source file is used to create two object files with different
+	preprocessor definitions. """
+	result = []
+	for dep in dependencies:
+		result.append(dep)
+		if (dep[0] == current):
+			depAdd = [additional, dep[1]]
+			result.append(depAdd)
+	return result
+
+def ExtractDependencies(input):
+	""" Create a list of dependencies from input list of lines
+	Each element contains the name of the object and a list of
+	files that it depends on.
+	Dependencies that contain "/usr/" are removed as they are system headers. """
+
+	deps = []
+	for line in input:
+		headersLine = line.startswith(" ") or line.startswith("\t")
+		line = line.strip()
+		isContinued = line.endswith("\\")
+		line = line.rstrip("\\ ")
+		fileNames = line.strip().split(" ")
+		if not headersLine:
+			# its a source file line, there may be headers too
+			sourceLine = fileNames[0].rstrip(":")
+			fileNames = fileNames[1:]
+			deps.append([sourceLine, []])
+		deps[-1][1].extend(header for header in fileNames if "/usr/" not in header)
+	return deps
+
+def TextFromDependencies(dependencies):
+	""" Convert a list of dependencies to text. """
+	text = ""
+	indentHeaders = "\t"
+	joinHeaders = continuationLineEnd + os.linesep + indentHeaders
+	for dep in dependencies:
+		object, headers = dep
+		text += object + ":"
+		for header in headers:
+			text += joinHeaders
+			text += header
+		if headers:
+			text += os.linesep
+	return text
+
+def UpdateDependencies(filepath, dependencies, comment=""):
+	""" Write a dependencies file if different from dependencies. """
+	FileGenerator.UpdateFile(os.path.abspath(filepath), comment.rstrip() + os.linesep +
+		TextFromDependencies(dependencies))
+
+def WriteDependencies(output, dependencies):
+	""" Write a list of dependencies out to a stream. """
+	output.write(TextFromDependencies(dependencies))
+
+if __name__ == "__main__":
+	""" Act as a filter that reformats input dependencies to one per line. """
+	inputLines = sys.stdin.readlines()
+	deps = ExtractDependencies(inputLines)
+	WriteDependencies(sys.stdout, deps)

scintilla/scripts/Face.py

@@ -37,6 +37,9 @@ def decodeParam(p):
 			name = nv
 	return type, name, value
 
+def IsEnumeration(t):
+	return t[:1].isupper()
+
 class Face:
 
 	def __init__(self):
@@ -44,6 +47,7 @@ class Face:
 		self.features = {}
 		self.values = {}
 		self.events = {}
+		self.aliases = {}
 
 	def ReadFromFile(self, name):
 		currentCategory = ""
@@ -117,4 +121,9 @@ class Face:
 							"Comment": currentComment }
 						self.order.append(name)
 						currentComment = []
+					elif featureType == "ali":
+						# Enumeration alias
+						name, value = featureVal.split("=", 1)
+						self.aliases[name] = value
+						currentComment = []
 

scintilla/scripts/GenerateCharacterCategory.py

@@ -17,17 +17,27 @@ def findCategories(filename):
 def updateCharacterCategory(filename):
     values = ["// Created with Python %s,  Unicode %s" % (
         platform.python_version(), unicodedata.unidata_version)]
-    category = unicodedata.category(chr(0))
+
     startRange = 0
+    category = unicodedata.category(chr(startRange))
+    table = []
     for ch in range(sys.maxunicode):
         uch = chr(ch)
-        if unicodedata.category(uch) != category:
+        current = unicodedata.category(uch)
+        if current != category:
             value = startRange * 32 + categories.index(category)
-            values.append("%d," % value)
+            table.append(value)
-            category = unicodedata.category(uch)
+            category = current
             startRange = ch
     value = startRange * 32 + categories.index(category)
-    values.append("%d," % value)
+    table.append(value)
+
+    # the sentinel value is used to simplify CharacterCategoryMap::Optimize()
+    category = 'Cn'
+    value = (sys.maxunicode + 1)*32 + categories.index(category)
+    table.append(value)
+
+    values.extend(["%d," % value for value in table])
 
     Regenerate(filename, "//", values)
 

scintilla/scripts/LexGen.py

@@ -1,10 +1,10 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 # LexGen.py - implemented 2002 by Neil Hodgson neilh@scintilla.org
 # Released to the public domain.
 
 # Regenerate the Scintilla source files that list all the lexers.
 # Should be run whenever a new lexer is added or removed.
-# Requires Python 2.5 or later
+# Requires Python 3.6 or later
 # Files are regenerated in place with templates stored in comments.
 # The format of generation comments is documented in FileGenerator.py.
 
@@ -13,7 +13,15 @@ from FileGenerator import Regenerate, UpdateLineInFile, \
     FindSectionInList
 import ScintillaData
 import HFacer
+import os
 import uuid
+import sys
+
+baseDirectory = os.path.dirname(os.path.dirname(ScintillaData.__file__))
+sys.path.append(baseDirectory)
+
+import win32.DepGen
+import gtk.DepGen
 
 def UpdateVersionNumbers(sci, root):
     UpdateLineInFile(root + "win32/ScintRes.rc", "#define VERSION_SCINTILLA",
@@ -110,11 +118,20 @@ def RegenerateXcodeProject(path, lexers, lexerReferences):
 
 def RegenerateAll(root):
 
+    scintillaBase = os.path.abspath(root)
+
     sci = ScintillaData.ScintillaData(root)
 
     Regenerate(root + "src/Catalogue.cxx", "//", sci.lexerModules)
     Regenerate(root + "win32/scintilla.mak", "#", sci.lexFiles)
 
+    startDir = os.getcwd()
+    os.chdir(os.path.join(scintillaBase, "win32"))
+    win32.DepGen.Generate()
+    os.chdir(os.path.join(scintillaBase, "gtk"))
+    gtk.DepGen.Generate()
+    os.chdir(startDir)
+
     RegenerateXcodeProject(root + "cocoa/ScintillaFramework/ScintillaFramework.xcodeproj/project.pbxproj",
         sci.lexFiles, sci.lexersXcode)
 

scintilla/src/CallTip.cxx

@@ -155,7 +155,7 @@ void CallTip::DrawChunk(Surface *surface, int &x, const char *s,
 				xEnd = NextTabPos(x);
 			} else {
 				std::string_view segText(s + startSeg, endSeg - startSeg);
-				xEnd = x + static_cast<int>(lround(surface->WidthText(font, segText)));
+				xEnd = x + static_cast<int>(std::lround(surface->WidthText(font, segText)));
 				if (draw) {
 					rcClient.left = static_cast<XYPOSITION>(x);
 					rcClient.right = static_cast<XYPOSITION>(xEnd);
@@ -176,7 +176,7 @@ int CallTip::PaintContents(Surface *surfaceWindow, bool draw) {
 	PRectangle rcClient(1.0f, 1.0f, rcClientSize.right - 1, rcClientSize.bottom - 1);
 
 	// To make a nice small call tip window, it is only sized to fit most normal characters without accents
-	const int ascent = static_cast<int>(lround(surfaceWindow->Ascent(font) - surfaceWindow->InternalLeading(font)));
+	const int ascent = static_cast<int>(std::round(surfaceWindow->Ascent(font) - surfaceWindow->InternalLeading(font)));
 
 	// For each line...
 	// Draw the definition in three parts: before highlight, highlighted, after highlight
@@ -280,7 +280,7 @@ PRectangle CallTip::CallTipStart(Sci::Position pos, Point pt, int textHeight, co
 	rectDown = PRectangle(0,0,0,0);
 	offsetMain = insetX;            // changed to right edge of any arrows
 	const int width = PaintContents(surfaceMeasure.get(), false) + insetX;
-	lineHeight = static_cast<int>(lround(surfaceMeasure->Height(font)));
+	lineHeight = static_cast<int>(std::lround(surfaceMeasure->Height(font)));
 
 	// The returned
 	// rectangle is aligned to the right edge of the last arrow encountered in

scintilla/src/CaseConvert.cxx

@@ -570,7 +570,7 @@ class CaseConverter : public ICaseConverter {
 	enum { maxConversionLength=6 };
 	struct ConversionString {
 		char conversion[maxConversionLength+1];
-		ConversionString() : conversion{} {
+		ConversionString() noexcept : conversion{} {
 		}
 	};
 	// Conversions are initially store in a vector of structs but then decomposed into
@@ -583,7 +583,7 @@ class CaseConverter : public ICaseConverter {
 		}
 		CharacterConversion(int character_, std::string_view conversion_) noexcept : character(character_) {
 			assert(conversion_.length() <= maxConversionLength);
-			std::copy(std::begin(conversion_), std::end(conversion_), conversion.conversion);
+			conversion_.copy(conversion.conversion, conversion_.length());
 		}
 		bool operator<(const CharacterConversion &other) const noexcept {
 			return character < other.character;
@@ -596,10 +596,10 @@ class CaseConverter : public ICaseConverter {
 	std::vector<ConversionString> conversions;
 
 public:
-	CaseConverter() {
+	CaseConverter() noexcept {
 	}
 	virtual ~CaseConverter() = default;
-	bool Initialised() const {
+	bool Initialised() const noexcept {
 		return !characters.empty();
 	}
 	void Add(int character, const char *conversion) {
@@ -776,7 +776,7 @@ void SetupConversions(enum CaseConversion conversion) {
 	}
 }
 
-CaseConverter *ConverterForConversion(enum CaseConversion conversion) {
+CaseConverter *ConverterForConversion(enum CaseConversion conversion) noexcept {
 	switch (conversion) {
 	case CaseConversionFold:
 		return &caseConvFold;

scintilla/src/Catalogue.cxx

@@ -102,6 +102,7 @@ int Scintilla_LinkLexers() {
 	LINK_LEXER(lmCsound);
 	LINK_LEXER(lmCss);
 	LINK_LEXER(lmD);
+	LINK_LEXER(lmDataflex);
 	LINK_LEXER(lmDiff);
 	LINK_LEXER(lmDMAP);
 	LINK_LEXER(lmDMIS);
@@ -197,6 +198,7 @@ int Scintilla_LinkLexers() {
 	LINK_LEXER(lmVerilog);
 	LINK_LEXER(lmVHDL);
 	LINK_LEXER(lmVisualProlog);
+	LINK_LEXER(lmX12);
 	LINK_LEXER(lmXML);
 	LINK_LEXER(lmYAML);
 

scintilla/src/CellBuffer.cxx

@@ -456,7 +456,7 @@ void UndoHistory::SetSavePoint() {
 	savePoint = currentAction;
 }
 
-bool UndoHistory::IsSavePoint() const {
+bool UndoHistory::IsSavePoint() const noexcept {
 	return savePoint == currentAction;
 }
 
@@ -480,7 +480,7 @@ int UndoHistory::TentativeSteps() {
 		return -1;
 }
 
-bool UndoHistory::CanUndo() const {
+bool UndoHistory::CanUndo() const noexcept {
 	return (currentAction > 0) && (maxAction > 0);
 }
 
@@ -505,7 +505,7 @@ void UndoHistory::CompletedUndoStep() {
 	currentAction--;
 }
 
-bool UndoHistory::CanRedo() const {
+bool UndoHistory::CanRedo() const noexcept {
 	return maxAction > currentAction;
 }
 
@@ -559,8 +559,10 @@ void CellBuffer::GetCharRange(char *buffer, Sci::Position position, Sci::Positio
 	if (position < 0)
 		return;
 	if ((position + lengthRetrieve) > substance.Length()) {
-		Platform::DebugPrintf("Bad GetCharRange %d for %d of %d\n", position,
+		Platform::DebugPrintf("Bad GetCharRange %.0f for %.0f of %.0f\n",
-		                      lengthRetrieve, substance.Length());
+				      static_cast<double>(position),
+				      static_cast<double>(lengthRetrieve),
+				      static_cast<double>(substance.Length()));
 		return;
 	}
 	substance.GetRange(buffer, position, lengthRetrieve);
@@ -580,8 +582,10 @@ void CellBuffer::GetStyleRange(unsigned char *buffer, Sci::Position position, Sc
 		return;
 	}
 	if ((position + lengthRetrieve) > style.Length()) {
-		Platform::DebugPrintf("Bad GetStyleRange %d for %d of %d\n", position,
+		Platform::DebugPrintf("Bad GetStyleRange %.0f for %.0f of %.0f\n",
-		                      lengthRetrieve, style.Length());
+				      static_cast<double>(position),
+				      static_cast<double>(lengthRetrieve),
+				      static_cast<double>(style.Length()));
 		return;
 	}
 	style.GetRange(reinterpret_cast<char *>(buffer), position, lengthRetrieve);
@@ -595,7 +599,7 @@ const char *CellBuffer::RangePointer(Sci::Position position, Sci::Position range
 	return substance.RangePointer(position, rangeLength);
 }
 
-Sci::Position CellBuffer::GapPosition() const {
+Sci::Position CellBuffer::GapPosition() const noexcept {
 	return substance.GapPosition();
 }
 
@@ -678,7 +682,6 @@ void CellBuffer::Allocate(Sci::Position newSize) {
 void CellBuffer::SetUTF8Substance(bool utf8Substance_) {
 	if (utf8Substance != utf8Substance_) {
 		utf8Substance = utf8Substance_;
-		ResetLineEnds();
 	}
 }
 
@@ -691,7 +694,7 @@ void CellBuffer::SetLineEndTypes(int utf8LineEnds_) {
 	}
 }
 
-bool CellBuffer::ContainsLineEnd(const char *s, Sci::Position length) const {
+bool CellBuffer::ContainsLineEnd(const char *s, Sci::Position length) const noexcept {
 	unsigned char chBeforePrev = 0;
 	unsigned char chPrev = 0;
 	for (Sci::Position i = 0; i < length; i++) {
@@ -756,7 +759,7 @@ Sci::Line CellBuffer::LineFromPositionIndex(Sci::Position pos, int lineCharacter
 	return plv->LineFromPositionIndex(pos, lineCharacterIndex);
 }
 
-bool CellBuffer::IsReadOnly() const {
+bool CellBuffer::IsReadOnly() const noexcept {
 	return readOnly;
 }
 
@@ -764,11 +767,11 @@ void CellBuffer::SetReadOnly(bool set) {
 	readOnly = set;
 }
 
-bool CellBuffer::IsLarge() const {
+bool CellBuffer::IsLarge() const noexcept {
 	return largeDocument;
 }
 
-bool CellBuffer::HasStyles() const {
+bool CellBuffer::HasStyles() const noexcept {
 	return hasStyles;
 }
 
@@ -776,7 +779,7 @@ void CellBuffer::SetSavePoint() {
 	uh.SetSavePoint();
 }
 
-bool CellBuffer::IsSavePoint() const {
+bool CellBuffer::IsSavePoint() const noexcept {
 	return uh.IsSavePoint();
 }
 
@@ -792,7 +795,7 @@ int CellBuffer::TentativeSteps() {
 	return uh.TentativeSteps();
 }
 
-bool CellBuffer::TentativeActive() const {
+bool CellBuffer::TentativeActive() const noexcept {
 	return uh.TentativeActive();
 }
 
@@ -806,7 +809,7 @@ void CellBuffer::RemoveLine(Sci::Line line) {
 	plv->RemoveLine(line);
 }
 
-bool CellBuffer::UTF8LineEndOverlaps(Sci::Position position) const {
+bool CellBuffer::UTF8LineEndOverlaps(Sci::Position position) const noexcept {
 	const unsigned char bytes[] = {
 		static_cast<unsigned char>(substance.ValueAt(position-2)),
 		static_cast<unsigned char>(substance.ValueAt(position-1)),
@@ -1134,7 +1137,7 @@ bool CellBuffer::SetUndoCollection(bool collectUndo) {
 	return collectingUndo;
 }
 
-bool CellBuffer::IsCollectingUndo() const {
+bool CellBuffer::IsCollectingUndo() const noexcept {
 	return collectingUndo;
 }
 
@@ -1155,7 +1158,7 @@ void CellBuffer::DeleteUndoHistory() {
 	uh.DeleteUndoHistory();
 }
 
-bool CellBuffer::CanUndo() const {
+bool CellBuffer::CanUndo() const noexcept {
 	return uh.CanUndo();
 }
 
@@ -1181,7 +1184,7 @@ void CellBuffer::PerformUndoStep() {
 	uh.CompletedUndoStep();
 }
 
-bool CellBuffer::CanRedo() const {
+bool CellBuffer::CanRedo() const noexcept {
 	return uh.CanRedo();
 }
 

scintilla/src/CellBuffer.h

@@ -81,7 +81,7 @@ public:
 	/// The save point is a marker in the undo stack where the container has stated that
 	/// the buffer was saved. Undo and redo can move over the save point.
 	void SetSavePoint();
-	bool IsSavePoint() const;
+	bool IsSavePoint() const noexcept;
 
 	// Tentative actions are used for input composition so that it can be undone cleanly
 	void TentativeStart();
@@ -91,11 +91,11 @@ public:
 
 	/// To perform an undo, StartUndo is called to retrieve the number of steps, then UndoStep is
 	/// called that many times. Similarly for redo.
-	bool CanUndo() const;
+	bool CanUndo() const noexcept;
 	int StartUndo();
 	const Action &GetUndoStep() const;
 	void CompletedUndoStep();
-	bool CanRedo() const;
+	bool CanRedo() const noexcept;
 	int StartRedo();
 	const Action &GetRedoStep() const;
 	void CompletedRedoStep();
@@ -121,7 +121,7 @@ private:
 
 	std::unique_ptr<ILineVector> plv;
 
-	bool UTF8LineEndOverlaps(Sci::Position position) const;
+	bool UTF8LineEndOverlaps(Sci::Position position) const noexcept;
 	bool UTF8IsCharacterBoundary(Sci::Position position) const;
 	void ResetLineEnds();
 	void RecalculateIndexLineStarts(Sci::Line lineFirst, Sci::Line lineLast);
@@ -148,14 +148,14 @@ public:
 	void GetStyleRange(unsigned char *buffer, Sci::Position position, Sci::Position lengthRetrieve) const;
 	const char *BufferPointer();
 	const char *RangePointer(Sci::Position position, Sci::Position rangeLength);
-	Sci::Position GapPosition() const;
+	Sci::Position GapPosition() const noexcept;
 
 	Sci::Position Length() const noexcept;
 	void Allocate(Sci::Position newSize);
 	void SetUTF8Substance(bool utf8Substance_);
-	int GetLineEndTypes() const { return utf8LineEnds; }
+	int GetLineEndTypes() const noexcept { return utf8LineEnds; }
 	void SetLineEndTypes(int utf8LineEnds_);
-	bool ContainsLineEnd(const char *s, Sci::Position length) const;
+	bool ContainsLineEnd(const char *s, Sci::Position length) const noexcept;
 	void SetPerLine(PerLine *pl);
 	int LineCharacterIndex() const noexcept;
 	void AllocateLineCharacterIndex(int lineCharacterIndex);
@@ -176,23 +176,23 @@ public:
 
 	const char *DeleteChars(Sci::Position position, Sci::Position deleteLength, bool &startSequence);
 
-	bool IsReadOnly() const;
+	bool IsReadOnly() const noexcept;
 	void SetReadOnly(bool set);
-	bool IsLarge() const;
+	bool IsLarge() const noexcept;
-	bool HasStyles() const;
+	bool HasStyles() const noexcept;
 
 	/// The save point is a marker in the undo stack where the container has stated that
 	/// the buffer was saved. Undo and redo can move over the save point.
 	void SetSavePoint();
-	bool IsSavePoint() const;
+	bool IsSavePoint() const noexcept;
 
 	void TentativeStart();
 	void TentativeCommit();
-	bool TentativeActive() const;
+	bool TentativeActive() const noexcept;
 	int TentativeSteps();
 
 	bool SetUndoCollection(bool collectUndo);
-	bool IsCollectingUndo() const;
+	bool IsCollectingUndo() const noexcept;
 	void BeginUndoAction();
 	void EndUndoAction();
 	void AddUndoAction(Sci::Position token, bool mayCoalesce);
@@ -200,11 +200,11 @@ public:
 
 	/// To perform an undo, StartUndo is called to retrieve the number of steps, then UndoStep is
 	/// called that many times. Similarly for redo.
-	bool CanUndo() const;
+	bool CanUndo() const noexcept;
 	int StartUndo();
 	const Action &GetUndoStep() const;
 	void PerformUndoStep();
-	bool CanRedo() const;
+	bool CanRedo() const noexcept;
 	int StartRedo();
 	const Action &GetRedoStep() const;
 	void PerformRedoStep();

scintilla/src/CharClassify.h

@@ -18,8 +18,8 @@ public:
 	void SetDefaultCharClasses(bool includeWordClass);
 	void SetCharClasses(const unsigned char *chars, cc newCharClass);
 	int GetCharsOfClass(cc characterClass, unsigned char *buffer) const;
-	cc GetClass(unsigned char ch) const { return static_cast<cc>(charClass[ch]);}
+	cc GetClass(unsigned char ch) const noexcept { return static_cast<cc>(charClass[ch]);}
-	bool IsWord(unsigned char ch) const { return static_cast<cc>(charClass[ch]) == ccWord;}
+	bool IsWord(unsigned char ch) const noexcept { return static_cast<cc>(charClass[ch]) == ccWord;}
 
 private:
 	enum { maxChar=256 };

scintilla/src/ContractionState.cxx

@@ -75,7 +75,6 @@ public:
 	bool HiddenLines() const override;
 
 	const char *GetFoldDisplayText(Sci::Line lineDoc) const override;
-	bool GetFoldDisplayTextShown(Sci::Line lineDoc) const override;
 	bool SetFoldDisplayText(Sci::Line lineDoc, const char *text) override;
 
 	bool GetExpanded(Sci::Line lineDoc) const override;
@@ -211,17 +210,25 @@ Sci::Line ContractionState<LINE>::DocFromDisplay(Sci::Line lineDisplay) const {
 
 template <typename LINE>
 void ContractionState<LINE>::InsertLines(Sci::Line lineDoc, Sci::Line lineCount) {
+	if (OneToOne()) {
+		linesInDocument += static_cast<LINE>(lineCount);
+	} else {
 		for (Sci::Line l = 0; l < lineCount; l++) {
 			InsertLine(lineDoc + l);
 		}
+	}
 	Check();
 }
 
 template <typename LINE>
 void ContractionState<LINE>::DeleteLines(Sci::Line lineDoc, Sci::Line lineCount) {
+	if (OneToOne()) {
+		linesInDocument -= static_cast<LINE>(lineCount);
+	} else {
 		for (Sci::Line l = 0; l < lineCount; l++) {
 			DeleteLine(lineDoc);
 		}
+	}
 	Check();
 }
 
@@ -277,17 +284,12 @@ const char *ContractionState<LINE>::GetFoldDisplayText(Sci::Line lineDoc) const
 	return foldDisplayTexts->ValueAt(lineDoc).get();
 }
 
-template <typename LINE>
-bool ContractionState<LINE>::GetFoldDisplayTextShown(Sci::Line lineDoc) const {
-	return !GetExpanded(lineDoc) && GetFoldDisplayText(lineDoc);
-}
-
 template <typename LINE>
 bool ContractionState<LINE>::SetFoldDisplayText(Sci::Line lineDoc, const char *text) {
 	EnsureData();
 	const char *foldText = foldDisplayTexts->ValueAt(lineDoc).get();
 	if (!foldText || !text || 0 != strcmp(text, foldText)) {
-		UniqueString uns = UniqueStringCopy(text);
+		UniqueString uns = IsNullOrEmpty(text) ? UniqueString() : UniqueStringCopy(text);
 		foldDisplayTexts->SetValueAt(lineDoc, std::move(uns));
 		Check();
 		return true;

scintilla/src/ContractionState.h

@@ -32,7 +32,6 @@ public:
 	virtual bool HiddenLines() const=0;
 
 	virtual const char *GetFoldDisplayText(Sci::Line lineDoc) const=0;
-	virtual bool GetFoldDisplayTextShown(Sci::Line lineDoc) const=0;
 	virtual bool SetFoldDisplayText(Sci::Line lineDoc, const char *text)=0;
 
 	virtual bool GetExpanded(Sci::Line lineDoc) const=0;

scintilla/src/DBCS.h

@@ -10,6 +10,14 @@
 
 namespace Scintilla {
 
+constexpr bool IsDBCSCodePage(int codePage) noexcept {
+	return codePage == 932
+	       || codePage == 936
+	       || codePage == 949
+	       || codePage == 950
+	       || codePage == 1361;
+}
+
 bool DBCSIsLeadByte(int codePage, char ch) noexcept;
 
 }

scintilla/src/Decoration.cxx

@@ -229,7 +229,7 @@ template <typename POS>
 void DecorationList<POS>::DeleteLexerDecorations() {
 	decorationList.erase(std::remove_if(decorationList.begin(), decorationList.end(),
 		[](const std::unique_ptr<Decoration<POS>> &deco) {
-		return deco->Indicator() < INDIC_CONTAINER;
+		return deco->Indicator() < INDICATOR_CONTAINER ;
 	}), decorationList.end());
 	current = nullptr;
 	SetView();
@@ -260,7 +260,7 @@ int DecorationList<POS>::AllOnFor(Sci::Position position) const {
 	int mask = 0;
 	for (const std::unique_ptr<Decoration<POS>> &deco : decorationList) {
 		if (deco->rs.ValueAt(static_cast<POS>(position))) {
-			if (deco->Indicator() < INDIC_IME) {
+			if (deco->Indicator() < INDICATOR_IME) {
 				mask |= 1 << deco->Indicator();
 			}
 		}

scintilla/src/Document.cxx

@@ -185,23 +185,23 @@ void Document::RemoveLine(Sci::Line line) {
 	}
 }
 
-LineMarkers *Document::Markers() const {
+LineMarkers *Document::Markers() const noexcept {
 	return static_cast<LineMarkers *>(perLineData[ldMarkers].get());
 }
 
-LineLevels *Document::Levels() const {
+LineLevels *Document::Levels() const noexcept {
 	return static_cast<LineLevels *>(perLineData[ldLevels].get());
 }
 
-LineState *Document::States() const {
+LineState *Document::States() const noexcept {
 	return static_cast<LineState *>(perLineData[ldState].get());
 }
 
-LineAnnotation *Document::Margins() const {
+LineAnnotation *Document::Margins() const noexcept {
 	return static_cast<LineAnnotation *>(perLineData[ldMargin].get());
 }
 
-LineAnnotation *Document::Annotations() const {
+LineAnnotation *Document::Annotations() const noexcept {
 	return static_cast<LineAnnotation *>(perLineData[ldAnnotation].get());
 }
 
@@ -307,7 +307,7 @@ void Document::TentativeUndo() {
 	}
 }
 
-int Document::GetMark(Sci::Line line) const {
+int Document::GetMark(Sci::Line line) const noexcept {
 	return Markers()->MarkValue(line);
 }
 
@@ -445,11 +445,11 @@ Sci::Position Document::VCHomePosition(Sci::Position position) const {
 		return startText;
 }
 
-Sci::Position Document::IndexLineStart(Sci::Line line, int lineCharacterIndex) const {
+Sci::Position Document::IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept {
 	return cb.IndexLineStart(line, lineCharacterIndex);
 }
 
-Sci::Line Document::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const {
+Sci::Line Document::LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept {
 	return cb.LineFromPositionIndex(pos, lineCharacterIndex);
 }
 
@@ -592,36 +592,51 @@ void Document::GetHighlightDelimiters(HighlightDelimiter &highlightDelimiter, Sc
 	highlightDelimiter.firstChangeableLineAfter = firstChangeableLineAfter;
 }
 
-Sci::Position Document::ClampPositionIntoDocument(Sci::Position pos) const {
+Sci::Position Document::ClampPositionIntoDocument(Sci::Position pos) const noexcept {
-	return std::clamp<Sci::Position>(pos, 0, Length());
+	return std::clamp<Sci::Position>(pos, 0, LengthNoExcept());
 }
 
-bool Document::IsCrLf(Sci::Position pos) const {
+bool Document::IsCrLf(Sci::Position pos) const noexcept {
 	if (pos < 0)
 		return false;
-	if (pos >= (Length() - 1))
+	if (pos >= (LengthNoExcept() - 1))
 		return false;
 	return (cb.CharAt(pos) == '\r') && (cb.CharAt(pos + 1) == '\n');
 }
 
-int Document::LenChar(Sci::Position pos) {
+int Document::LenChar(Sci::Position pos) const noexcept {
-	if (pos < 0) {
+	if (pos < 0 || pos >= LengthNoExcept()) {
+		// Returning 1 instead of 0 to defend against hanging with a loop that goes (or starts) out of bounds.
 		return 1;
 	} else if (IsCrLf(pos)) {
 		return 2;
-	} else if (SC_CP_UTF8 == dbcsCodePage) {
+	}
+
 	const unsigned char leadByte = cb.UCharAt(pos);
+	if (!dbcsCodePage || UTF8IsAscii(leadByte)) {
+		// Common case: ASCII character
+		return 1;
+	}
+	if (SC_CP_UTF8 == dbcsCodePage) {
 		const int widthCharBytes = UTF8BytesOfLead[leadByte];
-		const Sci::Position lengthDoc = Length();
+		unsigned char charBytes[UTF8MaxBytes] = { leadByte, 0, 0, 0 };
-		if ((pos + widthCharBytes) > lengthDoc)
+		for (int b = 1; b < widthCharBytes; b++) {
-			return static_cast<int>(lengthDoc - pos);
+			charBytes[b] = cb.UCharAt(pos + b);
-		else
+		}
-			return widthCharBytes;
+		const int utf8status = UTF8Classify(charBytes, widthCharBytes);
-	} else if (dbcsCodePage) {
+		if (utf8status & UTF8MaskInvalid) {
-		return IsDBCSLeadByteNoExcept(cb.CharAt(pos)) ? 2 : 1;
+			// Treat as invalid and use up just one byte
+			return 1;
+		} else {
+			return utf8status & UTF8MaskWidth;
+		}
+	} else {
+		if (IsDBCSLeadByteNoExcept(leadByte) && ((pos + 1) < LengthNoExcept())) {
+			return 2;
 		} else {
 			return 1;
 		}
+	}
 }
 
 bool Document::InGoodUTF8(Sci::Position pos, Sci::Position &start, Sci::Position &end) const noexcept {
@@ -656,15 +671,15 @@ bool Document::InGoodUTF8(Sci::Position pos, Sci::Position &start, Sci::Position
 // When lines are terminated with \r\n pairs which should be treated as one character.
 // When displaying DBCS text such as Japanese.
 // If moving, move the position in the indicated direction.
-Sci::Position Document::MovePositionOutsideChar(Sci::Position pos, Sci::Position moveDir, bool checkLineEnd) const {
+Sci::Position Document::MovePositionOutsideChar(Sci::Position pos, Sci::Position moveDir, bool checkLineEnd) const noexcept {
 	//Platform::DebugPrintf("NoCRLF %d %d\n", pos, moveDir);
 	// If out of range, just return minimum/maximum value.
 	if (pos <= 0)
 		return 0;
-	if (pos >= Length())
+	if (pos >= LengthNoExcept())
-		return Length();
+		return LengthNoExcept();
 
-	// PLATFORM_ASSERT(pos > 0 && pos < Length());
+	// PLATFORM_ASSERT(pos > 0 && pos < LengthNoExcept());
 	if (checkLineEnd && IsCrLf(pos - 1)) {
 		if (moveDir > 0)
 			return pos + 1;
@@ -691,7 +706,7 @@ Sci::Position Document::MovePositionOutsideChar(Sci::Position pos, Sci::Position
 		} else {
 			// Anchor DBCS calculations at start of line because start of line can
 			// not be a DBCS trail byte.
-			const Sci::Position posStartLine = LineStart(LineFromPosition(pos));
+			const Sci::Position posStartLine = cb.LineStart(cb.LineFromPosition(pos));
 			if (pos == posStartLine)
 				return pos;
 
@@ -812,8 +827,8 @@ bool Document::NextCharacter(Sci::Position &pos, int moveDir) const noexcept {
 	}
 }
 
-Document::CharacterExtracted Document::CharacterAfter(Sci::Position position) const {
+Document::CharacterExtracted Document::CharacterAfter(Sci::Position position) const noexcept {
-	if (position >= Length()) {
+	if (position >= LengthNoExcept()) {
 		return CharacterExtracted(unicodeReplacementChar, 0);
 	}
 	const unsigned char leadByte = cb.UCharAt(position);
@@ -834,7 +849,7 @@ Document::CharacterExtracted Document::CharacterAfter(Sci::Position position) co
 			return CharacterExtracted(UnicodeFromUTF8(charBytes), utf8status & UTF8MaskWidth);
 		}
 	} else {
-		if (IsDBCSLeadByteNoExcept(leadByte) && ((position + 1) < Length())) {
+		if (IsDBCSLeadByteNoExcept(leadByte) && ((position + 1) < LengthNoExcept())) {
 			return CharacterExtracted::DBCS(leadByte, cb.UCharAt(position + 1));
 		} else {
 			return CharacterExtracted(leadByte, 1);
@@ -842,7 +857,7 @@ Document::CharacterExtracted Document::CharacterAfter(Sci::Position position) co
 	}
 }
 
-Document::CharacterExtracted Document::CharacterBefore(Sci::Position position) const {
+Document::CharacterExtracted Document::CharacterBefore(Sci::Position position) const noexcept {
 	if (position <= 0) {
 		return CharacterExtracted(unicodeReplacementChar, 0);
 	}
@@ -903,7 +918,7 @@ Sci_Position SCI_METHOD Document::GetRelativePosition(Sci_Position positionStart
 	return pos;
 }
 
-Sci::Position Document::GetRelativePositionUTF16(Sci::Position positionStart, Sci::Position characterOffset) const {
+Sci::Position Document::GetRelativePositionUTF16(Sci::Position positionStart, Sci::Position characterOffset) const noexcept {
 	Sci::Position pos = positionStart;
 	if (dbcsCodePage) {
 		const int increment = (characterOffset > 0) ? 1 : -1;
@@ -918,7 +933,7 @@ Sci::Position Document::GetRelativePositionUTF16(Sci::Position positionStart, Sc
 		}
 	} else {
 		pos = positionStart + characterOffset;
-		if ((pos < 0) || (pos > Length()))
+		if ((pos < 0) || (pos > LengthNoExcept()))
 			return INVALID_POSITION;
 	}
 	return pos;
@@ -1168,7 +1183,7 @@ bool Document::DeleteChars(Sci::Position pos, Sci::Position len) {
 		return false;
 	if (len <= 0)
 		return false;
-	if ((pos + len) > Length())
+	if ((pos + len) > LengthNoExcept())
 		return false;
 	CheckReadOnly();
 	if (enteredModification != 0) {
@@ -1187,7 +1202,7 @@ bool Document::DeleteChars(Sci::Position pos, Sci::Position len) {
 			const char *text = cb.DeleteChars(pos, len, startSequence);
 			if (startSavePoint && cb.IsCollectingUndo())
 				NotifySavePoint(!startSavePoint);
-			if ((pos < Length()) || (pos == 0))
+			if ((pos < LengthNoExcept()) || (pos == 0))
 				ModifiedAt(pos);
 			else
 				ModifiedAt(pos-1);
@@ -1524,7 +1539,7 @@ Sci::Position Document::GetColumn(Sci::Position pos) {
 	return column;
 }
 
-Sci::Position Document::CountCharacters(Sci::Position startPos, Sci::Position endPos) const {
+Sci::Position Document::CountCharacters(Sci::Position startPos, Sci::Position endPos) const noexcept {
 	startPos = MovePositionOutsideChar(startPos, 1, false);
 	endPos = MovePositionOutsideChar(endPos, -1, false);
 	Sci::Position count = 0;
@@ -1536,7 +1551,7 @@ Sci::Position Document::CountCharacters(Sci::Position startPos, Sci::Position en
 	return count;
 }
 
-Sci::Position Document::CountUTF16(Sci::Position startPos, Sci::Position endPos) const {
+Sci::Position Document::CountUTF16(Sci::Position startPos, Sci::Position endPos) const noexcept {
 	startPos = MovePositionOutsideChar(startPos, 1, false);
 	endPos = MovePositionOutsideChar(endPos, -1, false);
 	Sci::Position count = 0;
@@ -1651,7 +1666,7 @@ void Document::ConvertLineEnds(int eolModeSet) {
 
 }
 
-int Document::Options() const {
+int Document::Options() const noexcept {
 	return (IsLarge() ? SC_DOCUMENTOPTION_TEXT_LARGE : 0) |
 		(cb.HasStyles() ? 0 : SC_DOCUMENTOPTION_STYLES_NONE);
 }
@@ -1695,19 +1710,11 @@ Sci::Position Document::ParaDown(Sci::Position pos) const {
 		return LineEnd(line-1);
 }
 
-bool Document::IsASCIIWordByte(unsigned char ch) const {
-	if (IsASCII(ch)) {
-		return charClass.GetClass(ch) == CharClassify::ccWord;
-	} else {
-		return false;
-	}
-}
-
 CharClassify::cc Document::WordCharacterClass(unsigned int ch) const {
 	if (dbcsCodePage && (!UTF8IsAscii(ch))) {
 		if (SC_CP_UTF8 == dbcsCodePage) {
 			// Use hard coded Unicode class
-			const CharacterCategory cc = CategoriseCharacter(ch);
+			const CharacterCategory cc = charMap.CategoryFor(ch);
 			switch (cc) {
 
 				// Separator, Line/Paragraph
@@ -1783,11 +1790,11 @@ Sci::Position Document::ExtendWordSelect(Sci::Position pos, int delta, bool only
 			pos -= ce.widthBytes;
 		}
 	} else {
-		if (!onlyWordCharacters && pos < Length()) {
+		if (!onlyWordCharacters && pos < LengthNoExcept()) {
 			const CharacterExtracted ce = CharacterAfter(pos);
 			ccStart = WordCharacterClass(ce.character);
 		}
-		while (pos < Length()) {
+		while (pos < LengthNoExcept()) {
 			const CharacterExtracted ce = CharacterAfter(pos);
 			if (WordCharacterClass(ce.character) != ccStart)
 				break;
@@ -1825,13 +1832,13 @@ Sci::Position Document::NextWordStart(Sci::Position pos, int delta) const {
 	} else {
 		CharacterExtracted ce = CharacterAfter(pos);
 		const CharClassify::cc ccStart = WordCharacterClass(ce.character);
-		while (pos < Length()) {
+		while (pos < LengthNoExcept()) {
 			ce = CharacterAfter(pos);
 			if (WordCharacterClass(ce.character) != ccStart)
 				break;
 			pos += ce.widthBytes;
 		}
-		while (pos < Length()) {
+		while (pos < LengthNoExcept()) {
 			ce = CharacterAfter(pos);
 			if (WordCharacterClass(ce.character) != CharClassify::ccSpace)
 				break;
@@ -1869,16 +1876,16 @@ Sci::Position Document::NextWordEnd(Sci::Position pos, int delta) const {
 			}
 		}
 	} else {
-		while (pos < Length()) {
+		while (pos < LengthNoExcept()) {
 			const CharacterExtracted ce = CharacterAfter(pos);
 			if (WordCharacterClass(ce.character) != CharClassify::ccSpace)
 				break;
 			pos += ce.widthBytes;
 		}
-		if (pos < Length()) {
+		if (pos < LengthNoExcept()) {
 			CharacterExtracted ce = CharacterAfter(pos);
 			const CharClassify::cc ccStart = WordCharacterClass(ce.character);
-			while (pos < Length()) {
+			while (pos < LengthNoExcept()) {
 				ce = CharacterAfter(pos);
 				if (WordCharacterClass(ce.character) != ccStart)
 					break;
@@ -1894,7 +1901,7 @@ Sci::Position Document::NextWordEnd(Sci::Position pos, int delta) const {
  * the previous character is of a different character class.
  */
 bool Document::IsWordStartAt(Sci::Position pos) const {
-	if (pos >= Length())
+	if (pos >= LengthNoExcept())
 		return false;
 	if (pos > 0) {
 		const CharacterExtracted cePos = CharacterAfter(pos);
@@ -1914,7 +1921,7 @@ bool Document::IsWordStartAt(Sci::Position pos) const {
 bool Document::IsWordEndAt(Sci::Position pos) const {
 	if (pos <= 0)
 		return false;
-	if (pos < Length()) {
+	if (pos < LengthNoExcept()) {
 		const CharacterExtracted cePos = CharacterAfter(pos);
 		const CharClassify::cc ccPos = WordCharacterClass(cePos.character);
 		const CharacterExtracted cePrev = CharacterBefore(pos);
@@ -1943,7 +1950,7 @@ bool Document::HasCaseFolder() const noexcept {
 	return pcf != nullptr;
 }
 
-void Document::SetCaseFolder(CaseFolder *pcf_) {
+void Document::SetCaseFolder(CaseFolder *pcf_) noexcept {
 	pcf.reset(pcf_);
 }
 
@@ -2138,7 +2145,7 @@ const char *Document::SubstituteByPosition(const char *text, Sci::Position *leng
 		return nullptr;
 }
 
-int Document::LineCharacterIndex() const {
+int Document::LineCharacterIndex() const noexcept {
 	return cb.LineCharacterIndex();
 }
 
@@ -2166,6 +2173,14 @@ int Document::GetCharsOfClass(CharClassify::cc characterClass, unsigned char *bu
     return charClass.GetCharsOfClass(characterClass, buffer);
 }
 
+void Document::SetCharacterCategoryOptimization(int countCharacters) {
+	charMap.Optimize(countCharacters);
+}
+
+int Document::CharacterCategoryOptimization() const noexcept {
+	return charMap.Size();
+}
+
 void SCI_METHOD Document::StartStyling(Sci_Position position) {
 	endStyled = position;
 }
@@ -2513,7 +2528,7 @@ Sci::Position Document::WordPartLeft(Sci::Position pos) const {
 
 Sci::Position Document::WordPartRight(Sci::Position pos) const {
 	CharacterExtracted ceStart = CharacterAfter(pos);
-	const Sci::Position length = Length();
+	const Sci::Position length = LengthNoExcept();
 	if (IsWordPartSeparator(ceStart.character)) {
 		while (pos < length && IsWordPartSeparator(CharacterAfter(pos).character))
 			pos += CharacterAfter(pos).widthBytes;
@@ -2555,14 +2570,14 @@ static constexpr bool IsLineEndChar(char c) noexcept {
 	return (c == '\n' || c == '\r');
 }
 
-Sci::Position Document::ExtendStyleRange(Sci::Position pos, int delta, bool singleLine) {
+Sci::Position Document::ExtendStyleRange(Sci::Position pos, int delta, bool singleLine) noexcept {
 	const int sStart = cb.StyleAt(pos);
 	if (delta < 0) {
 		while (pos > 0 && (cb.StyleAt(pos) == sStart) && (!singleLine || !IsLineEndChar(cb.CharAt(pos))))
 			pos--;
 		pos++;
 	} else {
-		while (pos < (Length()) && (cb.StyleAt(pos) == sStart) && (!singleLine || !IsLineEndChar(cb.CharAt(pos))))
+		while (pos < (LengthNoExcept()) && (cb.StyleAt(pos) == sStart) && (!singleLine || !IsLineEndChar(cb.CharAt(pos))))
 			pos++;
 	}
 	return pos;
@@ -2592,7 +2607,7 @@ static char BraceOpposite(char ch) noexcept {
 }
 
 // TODO: should be able to extend styled region to find matching brace
-Sci::Position Document::BraceMatch(Sci::Position position, Sci::Position /*maxReStyle*/) {
+Sci::Position Document::BraceMatch(Sci::Position position, Sci::Position /*maxReStyle*/) noexcept {
 	const char chBrace = CharAt(position);
 	const char chSeek = BraceOpposite(chBrace);
 	if (chSeek == '\0')
@@ -2603,7 +2618,7 @@ Sci::Position Document::BraceMatch(Sci::Position position, Sci::Position /*maxRe
 		direction = 1;
 	int depth = 1;
 	position = NextPosition(position, direction);
-	while ((position >= 0) && (position < Length())) {
+	while ((position >= 0) && (position < LengthNoExcept())) {
 		const char chAtPos = CharAt(position);
 		const int styAtPos = StyleIndexAt(position);
 		if ((position > GetEndStyled()) || (styAtPos == styBrace)) {
@@ -2659,7 +2674,7 @@ public:
 	Sci::Line lineRangeStart;
 	Sci::Line lineRangeEnd;
 	Sci::Line lineRangeBreak;
-	RESearchRange(const Document *doc_, Sci::Position minPos, Sci::Position maxPos) : doc(doc_) {
+	RESearchRange(const Document *doc_, Sci::Position minPos, Sci::Position maxPos) noexcept : doc(doc_) {
 		increment = (minPos <= maxPos) ? 1 : -1;
 
 		// Range endpoints should not be inside DBCS characters or between a CR and LF,
@@ -3058,17 +3073,9 @@ Sci::Position Cxx11RegexFindText(const Document *doc, Sci::Position minPos, Sci:
 
 		bool matched = false;
 		if (SC_CP_UTF8 == doc->dbcsCodePage) {
-			const std::string_view sv(s);
+			const std::wstring ws = WStringFromUTF8(s);
-			const size_t lenS = sv.length();
-			std::vector<wchar_t> ws(sv.length() + 1);
-#if WCHAR_T_IS_16
-			const size_t outLen = UTF16FromUTF8(sv, &ws[0], lenS);
-#else
-			const size_t outLen = UTF32FromUTF8(sv, reinterpret_cast<unsigned int *>(&ws[0]), lenS);
-#endif
-			ws[outLen] = 0;
 			std::wregex regexp;
-			regexp.assign(&ws[0], flagsRe);
+			regexp.assign(ws, flagsRe);
 			matched = MatchOnLines<UTF8Iterator>(doc, regexp, resr, search);
 
 		} else {

scintilla/src/Document.h

@@ -137,23 +137,23 @@ public:
 		firstChangeableLineAfter = -1;
 	}
 
-	bool NeedsDrawing(Sci::Line line) const {
+	bool NeedsDrawing(Sci::Line line) const noexcept {
 		return isEnabled && (line <= firstChangeableLineBefore || line >= firstChangeableLineAfter);
 	}
 
-	bool IsFoldBlockHighlighted(Sci::Line line) const {
+	bool IsFoldBlockHighlighted(Sci::Line line) const noexcept {
 		return isEnabled && beginFoldBlock != -1 && beginFoldBlock <= line && line <= endFoldBlock;
 	}
 
-	bool IsHeadOfFoldBlock(Sci::Line line) const {
+	bool IsHeadOfFoldBlock(Sci::Line line) const noexcept {
 		return beginFoldBlock == line && line < endFoldBlock;
 	}
 
-	bool IsBodyOfFoldBlock(Sci::Line line) const {
+	bool IsBodyOfFoldBlock(Sci::Line line) const noexcept {
 		return beginFoldBlock != -1 && beginFoldBlock < line && line < endFoldBlock;
 	}
 
-	bool IsTailOfFoldBlock(Sci::Line line) const {
+	bool IsTailOfFoldBlock(Sci::Line line) const noexcept {
 		return beginFoldBlock != -1 && beginFoldBlock < line && line == endFoldBlock;
 	}
 
@@ -230,6 +230,7 @@ private:
 	int refCount;
 	CellBuffer cb;
 	CharClassify charClass;
+	CharacterCategoryMap charMap;
 	std::unique_ptr<CaseFolder> pcf;
 	Sci::Position endStyled;
 	int styleClock;
@@ -245,11 +246,11 @@ private:
 	// ldSize is not real data - it is for dimensions and loops
 	enum lineData { ldMarkers, ldLevels, ldState, ldMargin, ldAnnotation, ldSize };
 	std::unique_ptr<PerLine> perLineData[ldSize];
-	LineMarkers *Markers() const;
+	LineMarkers *Markers() const noexcept;
-	LineLevels *Levels() const;
+	LineLevels *Levels() const noexcept;
-	LineState *States() const;
+	LineState *States() const noexcept;
-	LineAnnotation *Margins() const;
+	LineAnnotation *Margins() const noexcept;
-	LineAnnotation *Annotations() const;
+	LineAnnotation *Annotations() const noexcept;
 
 	bool matchesValid;
 	std::unique_ptr<RegexSearchBase> regex;
@@ -301,9 +302,9 @@ public:
 
 	int LineEndTypesSupported() const;
 	bool SetDBCSCodePage(int dbcsCodePage_);
-	int GetLineEndTypesAllowed() const { return cb.GetLineEndTypes(); }
+	int GetLineEndTypesAllowed() const noexcept { return cb.GetLineEndTypes(); }
 	bool SetLineEndTypesAllowed(int lineEndBitSet_);
-	int GetLineEndTypesActive() const { return cb.GetLineEndTypes(); }
+	int GetLineEndTypesActive() const noexcept { return cb.GetLineEndTypes(); }
 
 	int SCI_METHOD Version() const override {
 		return dvRelease4;
@@ -313,18 +314,18 @@ public:
 
 	Sci_Position SCI_METHOD LineFromPosition(Sci_Position pos) const override;
 	Sci::Line SciLineFromPosition(Sci::Position pos) const noexcept;	// Avoids casting LineFromPosition
-	Sci::Position ClampPositionIntoDocument(Sci::Position pos) const;
+	Sci::Position ClampPositionIntoDocument(Sci::Position pos) const noexcept;
-	bool ContainsLineEnd(const char *s, Sci::Position length) const { return cb.ContainsLineEnd(s, length); }
+	bool ContainsLineEnd(const char *s, Sci::Position length) const noexcept { return cb.ContainsLineEnd(s, length); }
-	bool IsCrLf(Sci::Position pos) const;
+	bool IsCrLf(Sci::Position pos) const noexcept;
-	int LenChar(Sci::Position pos);
+	int LenChar(Sci::Position pos) const noexcept;
 	bool InGoodUTF8(Sci::Position pos, Sci::Position &start, Sci::Position &end) const noexcept;
-	Sci::Position MovePositionOutsideChar(Sci::Position pos, Sci::Position moveDir, bool checkLineEnd=true) const;
+	Sci::Position MovePositionOutsideChar(Sci::Position pos, Sci::Position moveDir, bool checkLineEnd=true) const noexcept;
 	Sci::Position NextPosition(Sci::Position pos, int moveDir) const noexcept;
 	bool NextCharacter(Sci::Position &pos, int moveDir) const noexcept;	// Returns true if pos changed
-	Document::CharacterExtracted CharacterAfter(Sci::Position position) const;
+	Document::CharacterExtracted CharacterAfter(Sci::Position position) const noexcept;
-	Document::CharacterExtracted CharacterBefore(Sci::Position position) const;
+	Document::CharacterExtracted CharacterBefore(Sci::Position position) const noexcept;
 	Sci_Position SCI_METHOD GetRelativePosition(Sci_Position positionStart, Sci_Position characterOffset) const override;
-	Sci::Position GetRelativePositionUTF16(Sci::Position positionStart, Sci::Position characterOffset) const;
+	Sci::Position GetRelativePositionUTF16(Sci::Position positionStart, Sci::Position characterOffset) const noexcept;
 	int SCI_METHOD GetCharacterAndWidth(Sci_Position position, Sci_Position *pWidth) const override;
 	int SCI_METHOD CodePage() const override;
 	bool SCI_METHOD IsDBCSLeadByte(char ch) const override;
@@ -345,42 +346,42 @@ public:
 	void * SCI_METHOD ConvertToDocument() override;
 	Sci::Position Undo();
 	Sci::Position Redo();
-	bool CanUndo() const { return cb.CanUndo(); }
+	bool CanUndo() const noexcept { return cb.CanUndo(); }
-	bool CanRedo() const { return cb.CanRedo(); }
+	bool CanRedo() const noexcept { return cb.CanRedo(); }
 	void DeleteUndoHistory() { cb.DeleteUndoHistory(); }
 	bool SetUndoCollection(bool collectUndo) {
 		return cb.SetUndoCollection(collectUndo);
 	}
-	bool IsCollectingUndo() const { return cb.IsCollectingUndo(); }
+	bool IsCollectingUndo() const noexcept { return cb.IsCollectingUndo(); }
 	void BeginUndoAction() { cb.BeginUndoAction(); }
 	void EndUndoAction() { cb.EndUndoAction(); }
 	void AddUndoAction(Sci::Position token, bool mayCoalesce) { cb.AddUndoAction(token, mayCoalesce); }
 	void SetSavePoint();
-	bool IsSavePoint() const { return cb.IsSavePoint(); }
+	bool IsSavePoint() const noexcept { return cb.IsSavePoint(); }
 
 	void TentativeStart() { cb.TentativeStart(); }
 	void TentativeCommit() { cb.TentativeCommit(); }
 	void TentativeUndo();
-	bool TentativeActive() const { return cb.TentativeActive(); }
+	bool TentativeActive() const noexcept { return cb.TentativeActive(); }
 
 	const char * SCI_METHOD BufferPointer() override { return cb.BufferPointer(); }
 	const char *RangePointer(Sci::Position position, Sci::Position rangeLength) { return cb.RangePointer(position, rangeLength); }
-	Sci::Position GapPosition() const { return cb.GapPosition(); }
+	Sci::Position GapPosition() const noexcept { return cb.GapPosition(); }
 
 	int SCI_METHOD GetLineIndentation(Sci_Position line) override;
 	Sci::Position SetLineIndentation(Sci::Line line, Sci::Position indent);
 	Sci::Position GetLineIndentPosition(Sci::Line line) const;
 	Sci::Position GetColumn(Sci::Position pos);
-	Sci::Position CountCharacters(Sci::Position startPos, Sci::Position endPos) const;
+	Sci::Position CountCharacters(Sci::Position startPos, Sci::Position endPos) const noexcept;
-	Sci::Position CountUTF16(Sci::Position startPos, Sci::Position endPos) const;
+	Sci::Position CountUTF16(Sci::Position startPos, Sci::Position endPos) const noexcept;
 	Sci::Position FindColumn(Sci::Line line, Sci::Position column);
 	void Indent(bool forwards, Sci::Line lineBottom, Sci::Line lineTop);
 	static std::string TransformLineEnds(const char *s, size_t len, int eolModeWanted);
 	void ConvertLineEnds(int eolModeSet);
 	void SetReadOnly(bool set) { cb.SetReadOnly(set); }
-	bool IsReadOnly() const { return cb.IsReadOnly(); }
+	bool IsReadOnly() const noexcept { return cb.IsReadOnly(); }
-	bool IsLarge() const { return cb.IsLarge(); }
+	bool IsLarge() const noexcept { return cb.IsLarge(); }
-	int Options() const;
+	int Options() const noexcept;
 
 	void DelChar(Sci::Position pos);
 	void DelCharBack(Sci::Position pos);
@@ -394,7 +395,7 @@ public:
 	void GetStyleRange(unsigned char *buffer, Sci::Position position, Sci::Position lengthRetrieve) const {
 		cb.GetStyleRange(buffer, position, lengthRetrieve);
 	}
-	int GetMark(Sci::Line line) const;
+	int GetMark(Sci::Line line) const noexcept;
 	Sci::Line MarkerNext(Sci::Line lineStart, int mask) const;
 	int AddMark(Sci::Line line, int markerNum);
 	void AddMarkSet(Sci::Line line, int valueSet);
@@ -409,8 +410,8 @@ public:
 	bool IsLineEndPosition(Sci::Position position) const;
 	bool IsPositionInLineEnd(Sci::Position position) const;
 	Sci::Position VCHomePosition(Sci::Position position) const;
-	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const;
+	Sci::Position IndexLineStart(Sci::Line line, int lineCharacterIndex) const noexcept;
-	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const;
+	Sci::Line LineFromPositionIndex(Sci::Position pos, int lineCharacterIndex) const noexcept;
 
 	int SCI_METHOD SetLevel(Sci_Position line, int level) override;
 	int SCI_METHOD GetLevel(Sci_Position line) const override;
@@ -423,6 +424,7 @@ public:
 	Sci::Position NextWordStart(Sci::Position pos, int delta) const;
 	Sci::Position NextWordEnd(Sci::Position pos, int delta) const;
 	Sci_Position SCI_METHOD Length() const override { return cb.Length(); }
+	Sci::Position LengthNoExcept() const noexcept { return cb.Length(); }
 	void Allocate(Sci::Position newSize) { cb.Allocate(newSize); }
 
 	CharacterExtracted ExtractCharacter(Sci::Position position) const noexcept;
@@ -433,10 +435,10 @@ public:
 
 	bool MatchesWordOptions(bool word, bool wordStart, Sci::Position pos, Sci::Position length) const;
 	bool HasCaseFolder() const noexcept;
-	void SetCaseFolder(CaseFolder *pcf_);
+	void SetCaseFolder(CaseFolder *pcf_) noexcept;
 	Sci::Position FindText(Sci::Position minPos, Sci::Position maxPos, const char *search, int flags, Sci::Position *length);
 	const char *SubstituteByPosition(const char *text, Sci::Position *length);
-	int LineCharacterIndex() const;
+	int LineCharacterIndex() const noexcept;
 	void AllocateLineCharacterIndex(int lineCharacterIndex);
 	void ReleaseLineCharacterIndex(int lineCharacterIndex);
 	Sci::Line LinesTotal() const noexcept;
@@ -444,6 +446,8 @@ public:
 	void SetDefaultCharClasses(bool includeWordClass);
 	void SetCharClasses(const unsigned char *chars, CharClassify::cc newCharClass);
 	int GetCharsOfClass(CharClassify::cc characterClass, unsigned char *buffer) const;
+	void SetCharacterCategoryOptimization(int countCharacters);
+	int CharacterCategoryOptimization() const noexcept;
 	void SCI_METHOD StartStyling(Sci_Position position) override;
 	bool SCI_METHOD SetStyleFor(Sci_Position length, char style) override;
 	bool SCI_METHOD SetStyles(Sci_Position length, const char *styles) override;
@@ -479,17 +483,16 @@ public:
 	bool AddWatcher(DocWatcher *watcher, void *userData);
 	bool RemoveWatcher(DocWatcher *watcher, void *userData);
 
-	bool IsASCIIWordByte(unsigned char ch) const;
 	CharClassify::cc WordCharacterClass(unsigned int ch) const;
 	bool IsWordPartSeparator(unsigned int ch) const;
 	Sci::Position WordPartLeft(Sci::Position pos) const;
 	Sci::Position WordPartRight(Sci::Position pos) const;
-	Sci::Position ExtendStyleRange(Sci::Position pos, int delta, bool singleLine = false);
+	Sci::Position ExtendStyleRange(Sci::Position pos, int delta, bool singleLine) noexcept;
 	bool IsWhiteLine(Sci::Line line) const;
 	Sci::Position ParaUp(Sci::Position pos) const;
 	Sci::Position ParaDown(Sci::Position pos) const;
 	int IndentSize() const noexcept { return actualIndentInChars; }
-	Sci::Position BraceMatch(Sci::Position position, Sci::Position maxReStyle);
+	Sci::Position BraceMatch(Sci::Position position, Sci::Position maxReStyle) noexcept;
 
 private:
 	void NotifyModifyAttempt();

scintilla/src/EditModel.cxx

@@ -25,6 +25,8 @@
 #include "ILexer.h"
 #include "Scintilla.h"
 
+#include "CharacterCategory.h"
+
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"
@@ -86,3 +88,20 @@ bool EditModel::BidirectionalEnabled() const {
 bool EditModel::BidirectionalR2L() const {
 	return bidirectional == Bidirectional::bidiR2L;
 }
+
+void EditModel::SetDefaultFoldDisplayText(const char *text) {
+	defaultFoldDisplayText = IsNullOrEmpty(text) ? UniqueString() : UniqueStringCopy(text);
+}
+
+const char *EditModel::GetDefaultFoldDisplayText() const noexcept {
+	return defaultFoldDisplayText.get();
+}
+
+const char *EditModel::GetFoldDisplayText(Sci::Line lineDoc) const {
+	if (foldDisplayTextStyle == SC_FOLDDISPLAYTEXT_HIDDEN || pcs->GetExpanded(lineDoc)) {
+		return nullptr;
+	}
+
+	const char *text = pcs->GetFoldDisplayText(lineDoc);
+	return text ? text : defaultFoldDisplayText.get();
+}

scintilla/src/EditModel.h

@@ -37,11 +37,12 @@ public:
 	bool primarySelection;
 
 	enum IMEInteraction { imeWindowed, imeInline } imeInteraction;
-
+	enum class CharacterSource { directInput, tentativeInput, imeResult };
 	enum class Bidirectional { bidiDisabled, bidiL2R, bidiR2L  } bidirectional;
 
 	int foldFlags;
 	int foldDisplayTextStyle;
+	UniqueString defaultFoldDisplayText;
 	std::unique_ptr<IContractionState> pcs;
 	// Hotspot support
 	Range hotspot;
@@ -65,6 +66,9 @@ public:
 	virtual Range GetHotSpotRange() const noexcept = 0;
 	bool BidirectionalEnabled() const;
 	bool BidirectionalR2L() const;
+	void SetDefaultFoldDisplayText(const char *text);
+	const char *GetDefaultFoldDisplayText() const noexcept;
+	const char *GetFoldDisplayText(Sci::Line lineDoc) const;
 };
 
 }

scintilla/src/EditView.cxx

@@ -30,6 +30,7 @@
 #include "Scintilla.h"
 
 #include "CharacterSet.h"
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "IntegerRectangle.h"
 #include "UniqueString.h"
@@ -342,6 +343,34 @@ LineLayout *EditView::RetrieveLineLayout(Sci::Line lineNumber, const EditModel &
 		model.LinesOnScreen() + 1, model.pdoc->LinesTotal());
 }
 
+namespace {
+
+/**
+* Return the chDoc argument with case transformed as indicated by the caseForce argument.
+* chPrevious is needed for camel casing.
+* This only affects ASCII characters and is provided for languages with case-insensitive
+* ASCII keywords where the user wishes to view keywords in a preferred case.
+*/
+inline char CaseForce(Style::ecaseForced caseForce, char chDoc, char chPrevious) {
+	switch (caseForce) {
+	case Style::caseMixed:
+		return chDoc;
+	case Style::caseLower:
+		return MakeLowerCase(chDoc);
+	case Style::caseUpper:
+		return MakeUpperCase(chDoc);
+	case Style::caseCamel:
+	default:	// default should not occur, included to avoid warnings
+		if (IsUpperOrLowerCase(chDoc) && !IsUpperOrLowerCase(chPrevious)) {
+			return MakeUpperCase(chDoc);
+		} else {
+			return MakeLowerCase(chDoc);
+		}
+	}
+}
+
+}
+
 /**
 * Fill in the LineLayout data for the given line.
 * Copy the given @a line and its styles from the document into local arrays.
@@ -370,29 +399,16 @@ void EditView::LayoutLine(const EditModel &model, Sci::Line line, Surface *surfa
 			// Check base line layout
 			int styleByte = 0;
 			int numCharsInLine = 0;
+			char chPrevious = 0;
 			while (numCharsInLine < lineLength) {
 				const Sci::Position charInDoc = numCharsInLine + posLineStart;
 				const char chDoc = model.pdoc->CharAt(charInDoc);
 				styleByte = model.pdoc->StyleIndexAt(charInDoc);
 				allSame = allSame &&
 					(ll->styles[numCharsInLine] == styleByte);
-				if (vstyle.styles[ll->styles[numCharsInLine]].caseForce == Style::caseMixed)
 				allSame = allSame &&
-					(ll->chars[numCharsInLine] == chDoc);
+					(ll->chars[numCharsInLine] == CaseForce(vstyle.styles[styleByte].caseForce, chDoc, chPrevious));
-				else if (vstyle.styles[ll->styles[numCharsInLine]].caseForce == Style::caseLower)
+				chPrevious = chDoc;
-					allSame = allSame &&
-					(ll->chars[numCharsInLine] == MakeLowerCase(chDoc));
-				else if (vstyle.styles[ll->styles[numCharsInLine]].caseForce == Style::caseUpper)
-					allSame = allSame &&
-					(ll->chars[numCharsInLine] == MakeUpperCase(chDoc));
-				else	{ // Style::caseCamel
-					if ((model.pdoc->IsASCIIWordByte(ll->chars[numCharsInLine])) &&
-					  ((numCharsInLine == 0) || (!model.pdoc->IsASCIIWordByte(ll->chars[numCharsInLine - 1])))) {
-						allSame = allSame && (ll->chars[numCharsInLine] == MakeUpperCase(chDoc));
-					} else {
-						allSame = allSame && (ll->chars[numCharsInLine] == MakeLowerCase(chDoc));
-					}
-				}
 				numCharsInLine++;
 			}
 			allSame = allSame && (ll->styles[numCharsInLine] == styleByte);	// For eolFilled
@@ -424,26 +440,13 @@ void EditView::LayoutLine(const EditModel &model, Sci::Line line, Surface *surfa
 		model.pdoc->GetStyleRange(ll->styles.get(), posLineStart, lineLength);
 		const int numCharsBeforeEOL = static_cast<int>(model.pdoc->LineEnd(line) - posLineStart);
 		const int numCharsInLine = (vstyle.viewEOL) ? lineLength : numCharsBeforeEOL;
-		for (Sci::Position styleInLine = 0; styleInLine < numCharsInLine; styleInLine++) {
-			const unsigned char styleByte = ll->styles[styleInLine];
-			ll->styles[styleInLine] = styleByte;
-		}
 		const unsigned char styleByteLast = (lineLength > 0) ? ll->styles[lineLength - 1] : 0;
 		if (vstyle.someStylesForceCase) {
+			char chPrevious = 0;
 			for (int charInLine = 0; charInLine<lineLength; charInLine++) {
 				const char chDoc = ll->chars[charInLine];
-				if (vstyle.styles[ll->styles[charInLine]].caseForce == Style::caseUpper)
+				ll->chars[charInLine] = CaseForce(vstyle.styles[ll->styles[charInLine]].caseForce, chDoc, chPrevious);
-					ll->chars[charInLine] = MakeUpperCase(chDoc);
+				chPrevious = chDoc;
-				else if (vstyle.styles[ll->styles[charInLine]].caseForce == Style::caseLower)
-					ll->chars[charInLine] = MakeLowerCase(chDoc);
-				else if (vstyle.styles[ll->styles[charInLine]].caseForce == Style::caseCamel) {
-					if ((model.pdoc->IsASCIIWordByte(ll->chars[charInLine])) &&
-					  ((charInLine == 0) || (!model.pdoc->IsASCIIWordByte(ll->chars[charInLine - 1])))) {
-						ll->chars[charInLine] = MakeUpperCase(chDoc);
-					} else {
-						ll->chars[charInLine] = MakeLowerCase(chDoc);
-					}
-				}
 			}
 		}
 		ll->xHighlightGuide = 0;
@@ -700,7 +703,7 @@ Range EditView::RangeDisplayLine(Surface *surface, const EditModel &model, Sci::
 SelectionPosition EditView::SPositionFromLocation(Surface *surface, const EditModel &model, PointDocument pt, bool canReturnInvalid,
 	bool charPosition, bool virtualSpace, const ViewStyle &vs, const PRectangle rcClient) {
 	pt.x = pt.x - vs.textStart;
-	Sci::Line visibleLine = static_cast<int>(floor(pt.y / vs.lineHeight));
+	Sci::Line visibleLine = static_cast<int>(std::floor(pt.y / vs.lineHeight));
 	if (!canReturnInvalid && (visibleLine < 0))
 		visibleLine = 0;
 	const Sci::Line lineDoc = model.pcs->DocFromDisplay(visibleLine);
@@ -878,7 +881,7 @@ static void DrawTextBlob(Surface *surface, const ViewStyle &vsDraw, PRectangle r
 		surface->FillRectangle(rcSegment, textBack);
 	}
 	FontAlias ctrlCharsFont = vsDraw.styles[STYLE_CONTROLCHAR].font;
-	const int normalCharHeight = static_cast<int>(ceil(vsDraw.styles[STYLE_CONTROLCHAR].capitalHeight));
+	const int normalCharHeight = static_cast<int>(std::ceil(vsDraw.styles[STYLE_CONTROLCHAR].capitalHeight));
 	PRectangle rcCChar = rcSegment;
 	rcCChar.left = rcCChar.left + 1;
 	rcCChar.top = rcSegment.top + vsDraw.maxAscent - normalCharHeight;
@@ -1047,7 +1050,7 @@ void EditView::DrawEOL(Surface *surface, const EditModel &model, const ViewStyle
 		rcSegment.left = rcLine.left;
 	rcSegment.right = rcLine.right;
 
-	const bool fillRemainder = !lastSubLine || model.foldDisplayTextStyle == SC_FOLDDISPLAYTEXT_HIDDEN || !model.pcs->GetFoldDisplayTextShown(line);
+	const bool fillRemainder = !lastSubLine || !model.GetFoldDisplayText(line);
 	if (fillRemainder) {
 		// Fill the remainder of the line
 		FillLineRemainder(surface, model, vsDraw, ll, line, rcSegment, subLine);
@@ -1195,11 +1198,12 @@ void EditView::DrawFoldDisplayText(Surface *surface, const EditModel &model, con
 	if (!lastSubLine)
 		return;
 
-	if ((model.foldDisplayTextStyle == SC_FOLDDISPLAYTEXT_HIDDEN) || !model.pcs->GetFoldDisplayTextShown(line))
+	const char *text = model.GetFoldDisplayText(line);
+	if (!text)
 		return;
 
 	PRectangle rcSegment = rcLine;
-	const std::string_view foldDisplayText = model.pcs->GetFoldDisplayText(line);
+	const std::string_view foldDisplayText(text);
 	FontAlias fontText = vsDraw.styles[STYLE_FOLDDISPLAYTEXT].font;
 	const int widthFoldDisplayText = static_cast<int>(surface->WidthText(fontText, foldDisplayText));
 
@@ -1218,7 +1222,6 @@ void EditView::DrawFoldDisplayText(Surface *surface, const EditModel &model, con
 	rcSegment.right = rcSegment.left + static_cast<XYPOSITION>(widthFoldDisplayText);
 
 	const ColourOptional background = vsDraw.Background(model.pdoc->GetMark(line), model.caret.active, ll->containsCaret);
-	FontAlias textFont = vsDraw.styles[STYLE_FOLDDISPLAYTEXT].font;
 	ColourDesired textFore = vsDraw.styles[STYLE_FOLDDISPLAYTEXT].fore;
 	if (eolInSelection && (vsDraw.selColours.fore.isSet)) {
 		textFore = (eolInSelection == 1) ? vsDraw.selColours.fore : vsDraw.selAdditionalForeground;
@@ -1247,11 +1250,11 @@ void EditView::DrawFoldDisplayText(Surface *surface, const EditModel &model, con
 
 	if (phase & drawText) {
 		if (phasesDraw != phasesOne) {
-			surface->DrawTextTransparent(rcSegment, textFont,
+			surface->DrawTextTransparent(rcSegment, fontText,
 				rcSegment.top + vsDraw.maxAscent, foldDisplayText,
 				textFore);
 		} else {
-			surface->DrawTextNoClip(rcSegment, textFont,
+			surface->DrawTextNoClip(rcSegment, fontText,
 				rcSegment.top + vsDraw.maxAscent, foldDisplayText,
 				textFore, textBack);
 		}
@@ -1261,8 +1264,8 @@ void EditView::DrawFoldDisplayText(Surface *surface, const EditModel &model, con
 		if (model.foldDisplayTextStyle == SC_FOLDDISPLAYTEXT_BOXED) {
 			surface->PenColour(textFore);
 			PRectangle rcBox = rcSegment;
-			rcBox.left = round(rcSegment.left);
+			rcBox.left = std::round(rcSegment.left);
-			rcBox.right = round(rcSegment.right);
+			rcBox.right = std::round(rcSegment.right);
 			const IntegerRectangle ircBox(rcBox);
 			surface->MoveTo(ircBox.left, ircBox.top);
 			surface->LineTo(ircBox.left, ircBox.bottom);
@@ -1421,7 +1424,9 @@ void EditView::DrawCarets(Surface *surface, const EditModel &model, const ViewSt
 	for (size_t r = 0; (r<model.sel.Count()) || drawDrag; r++) {
 		const bool mainCaret = r == model.sel.Main();
 		SelectionPosition posCaret = (drawDrag ? model.posDrag : model.sel.Range(r).caret);
-		if ((vsDraw.IsBlockCaretStyle() || imeCaretBlockOverride) && !drawDrag && posCaret > model.sel.Range(r).anchor) {
+		if ((vsDraw.DrawCaretInsideSelection(model.inOverstrike, imeCaretBlockOverride)) && 
+			!drawDrag &&
+			posCaret > model.sel.Range(r).anchor) {
 			if (posCaret.VirtualSpace() > 0)
 				posCaret.SetVirtualSpace(posCaret.VirtualSpace() - 1);
 			else
@@ -1479,7 +1484,7 @@ void EditView::DrawCarets(Surface *surface, const EditModel &model, const ViewSt
 				const ViewStyle::CaretShape caretShape = drawDrag ? ViewStyle::CaretShape::line : vsDraw.CaretShapeForMode(model.inOverstrike);
 				if (drawDrag) {
 					/* Dragging text, use a line caret */
-					rcCaret.left = round(xposCaret - caretWidthOffset);
+					rcCaret.left = std::round(xposCaret - caretWidthOffset);
 					rcCaret.right = rcCaret.left + vsDraw.caretWidth;
 				} else if ((caretShape == ViewStyle::CaretShape::bar) && drawOverstrikeCaret) {
 					/* Overstrike (insert mode), use a modified bar caret */
@@ -1497,7 +1502,7 @@ void EditView::DrawCarets(Surface *surface, const EditModel &model, const ViewSt
 					}
 				} else {
 					/* Line caret */
-					rcCaret.left = round(xposCaret - caretWidthOffset);
+					rcCaret.left = std::round(xposCaret - caretWidthOffset);
 					rcCaret.right = rcCaret.left + vsDraw.caretWidth;
 				}
 				const ColourDesired caretColour = mainCaret ? vsDraw.caretcolour : vsDraw.additionalCaretColour;
@@ -1841,7 +1846,7 @@ void EditView::DrawForeground(Surface *surface, const EditModel &model, const Vi
 							indentCount <= (ll->positions[i + 1] - epsilon) / indentWidth;
 							indentCount++) {
 							if (indentCount > 0) {
-								const XYPOSITION xIndent = floor(indentCount * indentWidth);
+								const XYPOSITION xIndent = std::floor(indentCount * indentWidth);
 								DrawIndentGuide(surface, lineVisible, vsDraw.lineHeight, xIndent + xStart, rcSegment,
 									(ll->xHighlightGuide == xIndent));
 							}
@@ -1920,7 +1925,7 @@ void EditView::DrawForeground(Surface *surface, const EditModel &model, const Vi
 									indentCount <= (ll->positions[cpos + ts.start + 1] - epsilon) / indentWidth;
 									indentCount++) {
 									if (indentCount > 0) {
-										const XYPOSITION xIndent = floor(indentCount * indentWidth);
+										const XYPOSITION xIndent = std::floor(indentCount * indentWidth);
 										DrawIndentGuide(surface, lineVisible, vsDraw.lineHeight, xIndent + xStart, rcSegment,
 											(ll->xHighlightGuide == xIndent));
 									}
@@ -1997,7 +2002,7 @@ void EditView::DrawIndentGuidesOverEmpty(Surface *surface, const EditModel &mode
 		}
 
 		for (int indentPos = model.pdoc->IndentSize(); indentPos < indentSpace; indentPos += model.pdoc->IndentSize()) {
-			const XYPOSITION xIndent = floor(indentPos * vsDraw.spaceWidth);
+			const XYPOSITION xIndent = std::floor(indentPos * vsDraw.spaceWidth);
 			if (xIndent < xStartText) {
 				DrawIndentGuide(surface, lineVisible, vsDraw.lineHeight, xIndent + xStart, rcLine,
 					(ll->xHighlightGuide == xIndent));
@@ -2014,7 +2019,7 @@ void EditView::DrawLine(Surface *surface, const EditModel &model, const ViewStyl
 		return; // No further drawing
 	}
 
-	// See if something overrides the line background color.
+	// See if something overrides the line background colour.
 	const ColourOptional background = vsDraw.Background(model.pdoc->GetMark(line), model.caret.active, ll->containsCaret);
 
 	const Sci::Position posLineStart = model.pdoc->LineStart(line);

scintilla/src/Editor.cxx

@@ -30,6 +30,7 @@
 #include "Scintilla.h"
 
 #include "CharacterSet.h"
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"
@@ -99,10 +100,10 @@ Timer::Timer() noexcept :
 Idler::Idler() noexcept :
 		state(false), idlerID(0) {}
 
-static constexpr bool IsAllSpacesOrTabs(const char *s, unsigned int len) noexcept {
+static constexpr bool IsAllSpacesOrTabs(std::string_view sv) noexcept {
-	for (unsigned int i = 0; i < len; i++) {
+	for (const char ch : sv) {
 		// This is safe because IsSpaceOrTab() will return false for null terminators
-		if (!IsSpaceOrTab(s[i]))
+		if (!IsSpaceOrTab(ch))
 			return false;
 	}
 	return true;
@@ -1150,13 +1151,9 @@ slop | strict | jumps | even | Caret can go to the margin                 | When
 
 Editor::XYScrollPosition Editor::XYScrollToMakeVisible(const SelectionRange &range, const XYScrollOptions options) {
 	const PRectangle rcClient = GetTextRectangle();
-	Point pt = LocationFromPosition(range.caret);
-	Point ptAnchor = LocationFromPosition(range.anchor);
 	const Point ptOrigin = GetVisibleOriginInMain();
-	pt.x += ptOrigin.x;
+	const Point pt = LocationFromPosition(range.caret) + ptOrigin;
-	pt.y += ptOrigin.y;
+	const Point ptAnchor = LocationFromPosition(range.anchor) + ptOrigin;
-	ptAnchor.x += ptOrigin.x;
-	ptAnchor.y += ptOrigin.y;
 	const Point ptBottomCaret(pt.x, pt.y + vs.lineHeight - 1);
 
 	XYScrollPosition newXY(xOffset, topLine);
@@ -1894,7 +1891,7 @@ void Editor::AddChar(char ch) {
 	char s[2];
 	s[0] = ch;
 	s[1] = '\0';
-	AddCharUTF(s, 1);
+	InsertCharacter(std::string_view(s, 1), CharacterSource::directInput);
 }
 
 void Editor::FilterSelections() {
@@ -1904,8 +1901,11 @@ void Editor::FilterSelections() {
 	}
 }
 
-// AddCharUTF inserts an array of bytes which may or may not be in UTF-8.
+// InsertCharacter inserts a character encoded in document code page.
-void Editor::AddCharUTF(const char *s, unsigned int len, bool treatAsDBCS) {
+void Editor::InsertCharacter(std::string_view sv, CharacterSource charSource) {
+	if (sv.empty()) {
+		return;
+	}
 	FilterSelections();
 	{
 		UndoGroup ug(pdoc, (sel.Count() > 1) || !sel.Empty() || inOverstrike);
@@ -1943,7 +1943,7 @@ void Editor::AddCharUTF(const char *s, unsigned int len, bool treatAsDBCS) {
 					}
 				}
 				positionInsert = RealizeVirtualSpace(positionInsert, currentSel->caret.VirtualSpace());
-				const Sci::Position lengthInserted = pdoc->InsertString(positionInsert, s, len);
+				const Sci::Position lengthInserted = pdoc->InsertString(positionInsert, sv.data(), sv.length());
 				if (lengthInserted > 0) {
 					currentSel->caret.SetPosition(positionInsert + lengthInserted);
 					currentSel->anchor.SetPosition(positionInsert + lengthInserted);
@@ -1972,30 +1972,33 @@ void Editor::AddCharUTF(const char *s, unsigned int len, bool treatAsDBCS) {
 	// Avoid blinking during rapid typing:
 	ShowCaretAtCurrentPosition();
 	if ((caretSticky == SC_CARETSTICKY_OFF) ||
-		((caretSticky == SC_CARETSTICKY_WHITESPACE) && !IsAllSpacesOrTabs(s, len))) {
+		((caretSticky == SC_CARETSTICKY_WHITESPACE) && !IsAllSpacesOrTabs(sv))) {
 		SetLastXChosen();
 	}
 
-	if (treatAsDBCS) {
+	int ch = static_cast<unsigned char>(sv[0]);
-		NotifyChar((static_cast<unsigned char>(s[0]) << 8) |
+	if (pdoc->dbcsCodePage != SC_CP_UTF8) {
-		        static_cast<unsigned char>(s[1]));
+		if (sv.length() > 1) {
-	} else if (len > 0) {
+			// DBCS code page or DBCS font character set.
-		int byte = static_cast<unsigned char>(s[0]);
+			ch = (ch << 8) | static_cast<unsigned char>(sv[1]);
-		if ((byte < 0xC0) || (1 == len)) {
+		}
+	} else {
+		if ((ch < 0xC0) || (1 == sv.length())) {
 			// Handles UTF-8 characters between 0x01 and 0x7F and single byte
 			// characters when not in UTF-8 mode.
 			// Also treats \0 and naked trail bytes 0x80 to 0xBF as valid
 			// characters representing themselves.
 		} else {
 			unsigned int utf32[1] = { 0 };
-			UTF32FromUTF8(std::string_view(s, len), utf32, std::size(utf32));
+			UTF32FromUTF8(sv, utf32, std::size(utf32));
-			byte = utf32[0];
+			ch = utf32[0];
 		}
-		NotifyChar(byte);
 	}
+	NotifyChar(ch, charSource);
 
-	if (recordingMacro) {
+	if (recordingMacro && charSource != CharacterSource::tentativeInput) {
-		NotifyMacroRecord(SCI_REPLACESEL, 0, reinterpret_cast<sptr_t>(s));
+		std::string copy(sv); // ensure NUL-terminated
+		NotifyMacroRecord(SCI_REPLACESEL, 0, reinterpret_cast<sptr_t>(copy.data()));
 	}
 }
 
@@ -2339,10 +2342,11 @@ void Editor::NotifyErrorOccurred(Document *, void *, int status) {
 	errorStatus = status;
 }
 
-void Editor::NotifyChar(int ch) {
+void Editor::NotifyChar(int ch, CharacterSource charSource) {
 	SCNotification scn = {};
 	scn.nmhdr.code = SCN_CHARADDED;
 	scn.ch = ch;
+	scn.characterSource = static_cast<int>(charSource);
 	NotifyParent(scn);
 }
 
@@ -2662,12 +2666,16 @@ void Editor::NotifyModified(Document *, DocModification mh, void *) {
 			}
 
 			if (paintState == notPainting && !CanDeferToLastStep(mh)) {
+				if (SynchronousStylingToVisible()) {
 					QueueIdleWork(WorkNeeded::workStyle, pdoc->Length());
+				}
 				Redraw();
 			}
 		} else {
 			if (paintState == notPainting && mh.length && !CanEliminate(mh)) {
+				if (SynchronousStylingToVisible()) {
 					QueueIdleWork(WorkNeeded::workStyle, mh.position + mh.length);
+				}
 				InvalidateRange(mh.position, mh.position + mh.length);
 			}
 		}
@@ -3084,7 +3092,7 @@ void Editor::NewLine() {
 	for (size_t i = 0; i < countInsertions; i++) {
 		const char *eol = StringFromEOLMode(pdoc->eolMode);
 		while (*eol) {
-			NotifyChar(*eol);
+			NotifyChar(*eol, CharacterSource::directInput);
 			if (recordingMacro) {
 				char txt[2];
 				txt[0] = *eol;
@@ -4170,9 +4178,10 @@ void Editor::GoToLine(Sci::Line lineNo) {
 }
 
 static bool Close(Point pt1, Point pt2, Point threshold) noexcept {
-	if (std::abs(pt1.x - pt2.x) > threshold.x)
+	const Point ptDifference = pt2 - pt1;
+	if (std::abs(ptDifference.x) > threshold.x)
 		return false;
-	if (std::abs(pt1.y - pt2.y) > threshold.y)
+	if (std::abs(ptDifference.y) > threshold.y)
 		return false;
 	return true;
 }
@@ -4262,9 +4271,8 @@ void Editor::DisplayCursor(Window::Cursor c) {
 }
 
 bool Editor::DragThreshold(Point ptStart, Point ptNow) {
-	const XYPOSITION xMove = ptStart.x - ptNow.x;
+	const Point ptDiff = ptStart - ptNow;
-	const XYPOSITION yMove = ptStart.y - ptNow.y;
+	const XYPOSITION distanceSquared = ptDiff.x * ptDiff.x + ptDiff.y * ptDiff.y;
-	const XYPOSITION distanceSquared = xMove * xMove + yMove * yMove;
 	return distanceSquared > 16.0f;
 }
 
@@ -4744,7 +4752,7 @@ Range Editor::GetHotSpotRange() const noexcept {
 }
 
 void Editor::ButtonMoveWithModifiers(Point pt, unsigned int, int modifiers) {
-	if ((ptMouseLast.x != pt.x) || (ptMouseLast.y != pt.y)) {
+	if (ptMouseLast != pt) {
 		DwellEnd(true);
 	}
 
@@ -5068,7 +5076,7 @@ void Editor::StyleToPositionInView(Sci::Position pos) {
 }
 
 Sci::Position Editor::PositionAfterMaxStyling(Sci::Position posMax, bool scrolling) const {
-	if ((idleStyling == SC_IDLESTYLING_NONE) || (idleStyling == SC_IDLESTYLING_AFTERVISIBLE)) {
+	if (SynchronousStylingToVisible()) {
 		// Both states do not limit styling
 		return posMax;
 	}
@@ -5790,11 +5798,10 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 			if (wParam == 0)
 				return 0;
 			char *ptr = CharPtrFromSPtr(lParam);
-			size_t iChar = 0;
+			const Sci_Position len = std::min<Sci_Position>(wParam - 1, pdoc->Length());
-			for (; iChar < wParam - 1; iChar++)
+			pdoc->GetCharRange(ptr, 0, len);
-				ptr[iChar] = pdoc->CharAt(iChar);
+			ptr[len] = '\0';
-			ptr[iChar] = '\0';
+			return len;
-			return iChar;
 		}
 
 	case SCI_SETTEXT: {
@@ -5882,15 +5889,14 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 				pdoc->LineStart(static_cast<Sci::Line>(wParam));
 			const Sci::Position lineEnd =
 				pdoc->LineStart(static_cast<Sci::Line>(wParam + 1));
+			// not NUL terminated
+			const Sci::Position len = lineEnd - lineStart;
 			if (lParam == 0) {
-				return lineEnd - lineStart;
+				return len;
 			}
 			char *ptr = CharPtrFromSPtr(lParam);
-			Sci::Position iPlace = 0;
+			pdoc->GetCharRange(ptr, lineStart, len);
-			for (Sci::Position iChar = lineStart; iChar < lineEnd; iChar++) {
+			return len;
-				ptr[iPlace++] = pdoc->CharAt(iChar);
-			}
-			return iPlace;
 		}
 
 	case SCI_GETLINECOUNT:
@@ -5924,10 +5930,10 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 				return selectedText.LengthWithTerminator();
 			} else {
 				char *ptr = CharPtrFromSPtr(lParam);
-				size_t iChar = 0;
+				size_t iChar = selectedText.Length();
-				if (selectedText.Length()) {
+				if (iChar) {
-					for (; iChar < selectedText.LengthWithTerminator(); iChar++)
+					memcpy(ptr, selectedText.Data(), iChar);
-						ptr[iChar] = selectedText.Data()[iChar];
+					ptr[iChar++] = '\0';
 				} else {
 					ptr[0] = '\0';
 				}
@@ -6252,6 +6258,13 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		pdoc->SetDefaultCharClasses(true);
 		break;
 
+	case SCI_SETCHARACTERCATEGORYOPTIMIZATION:
+		pdoc->SetCharacterCategoryOptimization(static_cast<int>(wParam));
+		break;
+
+	case SCI_GETCHARACTERCATEGORYOPTIMIZATION:
+		return pdoc->CharacterCategoryOptimization();
+
 	case SCI_GETLENGTH:
 		return pdoc->Length();
 
@@ -6349,8 +6362,8 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 			if (lParam == 0)
 				return 0;
 			Sci_TextRange *tr = static_cast<Sci_TextRange *>(PtrFromSPtr(lParam));
-			int iPlace = 0;
+			Sci::Position iPlace = 0;
-			for (long iChar = tr->chrg.cpMin; iChar < tr->chrg.cpMax; iChar++) {
+			for (Sci::Position iChar = tr->chrg.cpMin; iChar < tr->chrg.cpMax; iChar++) {
 				tr->lpstrText[iPlace++] = pdoc->CharAt(iChar);
 				tr->lpstrText[iPlace++] = pdoc->StyleAt(iChar);
 			}
@@ -6427,11 +6440,9 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 			}
 			PLATFORM_ASSERT(wParam > 0);
 			char *ptr = CharPtrFromSPtr(lParam);
-			unsigned int iPlace = 0;
+			const Sci::Position len = std::min<uptr_t>(lineEnd - lineStart, wParam - 1);
-			for (Sci::Position iChar = lineStart; iChar < lineEnd && iPlace < wParam - 1; iChar++) {
+			pdoc->GetCharRange(ptr, lineStart, len);
-				ptr[iPlace++] = pdoc->CharAt(iChar);
+			ptr[len] = '\0';
-			}
-			ptr[iPlace] = '\0';
 			return sel.MainCaret() - lineStart;
 		}
 
@@ -7193,6 +7204,17 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		Redraw();
 		break;
 
+	case SCI_FOLDDISPLAYTEXTGETSTYLE:
+		return foldDisplayTextStyle;
+
+	case SCI_SETDEFAULTFOLDDISPLAYTEXT:
+		SetDefaultFoldDisplayText(CharPtrFromSPtr(lParam));
+		Redraw();
+		break;
+
+	case SCI_GETDEFAULTFOLDDISPLAYTEXT:
+		return StringResult(lParam, GetDefaultFoldDisplayText());
+
 	case SCI_TOGGLEFOLD:
 		FoldLine(static_cast<Sci::Line>(wParam), SC_FOLDACTION_TOGGLE);
 		break;
@@ -7302,7 +7324,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		return vs.caretcolour.AsInteger();
 
 	case SCI_SETCARETSTYLE:
-		if (wParam <= (CARETSTYLE_BLOCK | CARETSTYLE_OVERSTRIKE_BLOCK))
+		if (wParam <= (CARETSTYLE_BLOCK | CARETSTYLE_OVERSTRIKE_BLOCK | CARETSTYLE_BLOCK_AFTER))
 			vs.caretStyle = static_cast<int>(wParam);
 		else
 			/* Default to the line caret */
@@ -7336,7 +7358,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		break;
 
 	case SCI_INDICSETSTYLE:
-		if (wParam <= INDIC_MAX) {
+		if (wParam <= INDICATOR_MAX) {
 			vs.indicators[wParam].sacNormal.style = static_cast<int>(lParam);
 			vs.indicators[wParam].sacHover.style = static_cast<int>(lParam);
 			InvalidateStyleRedraw();
@@ -7344,10 +7366,10 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		break;
 
 	case SCI_INDICGETSTYLE:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].sacNormal.style : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].sacNormal.style : 0;
 
 	case SCI_INDICSETFORE:
-		if (wParam <= INDIC_MAX) {
+		if (wParam <= INDICATOR_MAX) {
 			vs.indicators[wParam].sacNormal.fore = ColourDesired(static_cast<int>(lParam));
 			vs.indicators[wParam].sacHover.fore = ColourDesired(static_cast<int>(lParam));
 			InvalidateStyleRedraw();
@@ -7355,67 +7377,67 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		break;
 
 	case SCI_INDICGETFORE:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].sacNormal.fore.AsInteger() : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].sacNormal.fore.AsInteger() : 0;
 
 	case SCI_INDICSETHOVERSTYLE:
-		if (wParam <= INDIC_MAX) {
+		if (wParam <= INDICATOR_MAX) {
 			vs.indicators[wParam].sacHover.style = static_cast<int>(lParam);
 			InvalidateStyleRedraw();
 		}
 		break;
 
 	case SCI_INDICGETHOVERSTYLE:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].sacHover.style : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].sacHover.style : 0;
 
 	case SCI_INDICSETHOVERFORE:
-		if (wParam <= INDIC_MAX) {
+		if (wParam <= INDICATOR_MAX) {
 			vs.indicators[wParam].sacHover.fore = ColourDesired(static_cast<int>(lParam));
 			InvalidateStyleRedraw();
 		}
 		break;
 
 	case SCI_INDICGETHOVERFORE:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].sacHover.fore.AsInteger() : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].sacHover.fore.AsInteger() : 0;
 
 	case SCI_INDICSETFLAGS:
-		if (wParam <= INDIC_MAX) {
+		if (wParam <= INDICATOR_MAX) {
 			vs.indicators[wParam].SetFlags(static_cast<int>(lParam));
 			InvalidateStyleRedraw();
 		}
 		break;
 
 	case SCI_INDICGETFLAGS:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].Flags() : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].Flags() : 0;
 
 	case SCI_INDICSETUNDER:
-		if (wParam <= INDIC_MAX) {
+		if (wParam <= INDICATOR_MAX) {
 			vs.indicators[wParam].under = lParam != 0;
 			InvalidateStyleRedraw();
 		}
 		break;
 
 	case SCI_INDICGETUNDER:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].under : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].under : 0;
 
 	case SCI_INDICSETALPHA:
-		if (wParam <= INDIC_MAX && lParam >=0 && lParam <= 255) {
+		if (wParam <= INDICATOR_MAX && lParam >=0 && lParam <= 255) {
 			vs.indicators[wParam].fillAlpha = static_cast<int>(lParam);
 			InvalidateStyleRedraw();
 		}
 		break;
 
 	case SCI_INDICGETALPHA:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].fillAlpha : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].fillAlpha : 0;
 
 	case SCI_INDICSETOUTLINEALPHA:
-		if (wParam <= INDIC_MAX && lParam >=0 && lParam <= 255) {
+		if (wParam <= INDICATOR_MAX && lParam >=0 && lParam <= 255) {
 			vs.indicators[wParam].outlineAlpha = static_cast<int>(lParam);
 			InvalidateStyleRedraw();
 		}
 		break;
 
 	case SCI_INDICGETOUTLINEALPHA:
-		return (wParam <= INDIC_MAX) ? vs.indicators[wParam].outlineAlpha : 0;
+		return (wParam <= INDICATOR_MAX) ? vs.indicators[wParam].outlineAlpha : 0;
 
 	case SCI_SETINDICATORCURRENT:
 		pdoc->DecorationSetCurrentIndicator(static_cast<int>(wParam));
@@ -7550,7 +7572,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		break;
 
 	case SCI_BRACEHIGHLIGHTINDICATOR:
-		if (lParam >= 0 && lParam <= INDIC_MAX) {
+		if (lParam >= 0 && lParam <= INDICATOR_MAX) {
 			vs.braceHighlightIndicatorSet = wParam != 0;
 			vs.braceHighlightIndicator = static_cast<int>(lParam);
 		}
@@ -7561,7 +7583,7 @@ sptr_t Editor::WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) {
 		break;
 
 	case SCI_BRACEBADLIGHTINDICATOR:
-		if (lParam >= 0 && lParam <= INDIC_MAX) {
+		if (lParam >= 0 && lParam <= INDICATOR_MAX) {
 			vs.braceBadLightIndicatorSet = wParam != 0;
 			vs.braceBadLightIndicator = static_cast<int>(lParam);
 		}

scintilla/src/Editor.h

@@ -260,7 +260,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	Editor(Editor &&) = delete;
 	Editor &operator=(const Editor &) = delete;
 	Editor &operator=(Editor &&) = delete;
-	~Editor() override;
+	// ~Editor() in public section
 	virtual void Initialise() = 0;
 	virtual void Finalise();
 
@@ -393,7 +393,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	Sci::Position RealizeVirtualSpace(Sci::Position position, Sci::Position virtualSpace);
 	SelectionPosition RealizeVirtualSpace(const SelectionPosition &position);
 	void AddChar(char ch);
-	virtual void AddCharUTF(const char *s, unsigned int len, bool treatAsDBCS=false);
+	virtual void InsertCharacter(std::string_view sv, CharacterSource charSource);
 	void ClearBeforeTentativeStart();
 	void InsertPaste(const char *text, Sci::Position len);
 	enum PasteShape { pasteStream=0, pasteRectangular = 1, pasteLine = 2 };
@@ -421,7 +421,7 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	virtual int GetCtrlID() { return ctrlID; }
 	virtual void NotifyParent(SCNotification scn) = 0;
 	virtual void NotifyStyleToNeeded(Sci::Position endStyleNeeded);
-	void NotifyChar(int ch);
+	void NotifyChar(int ch, CharacterSource charSource);
 	void NotifySavePoint(bool isSavePoint);
 	void NotifyModifyAttempt();
 	virtual void NotifyDoubleClick(Point pt, int modifiers);
@@ -430,7 +430,6 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	void NotifyHotSpotReleaseClick(Sci::Position position, int modifiers);
 	bool NotifyUpdateUI();
 	void NotifyPainted();
-	void NotifyScrolled();
 	void NotifyIndicatorClick(bool click, Sci::Position position, int modifiers);
 	bool NotifyMarginClick(Point pt, int modifiers);
 	bool NotifyMarginRightClick(Point pt, int modifiers);
@@ -523,6 +522,9 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	Sci::Position PositionAfterMaxStyling(Sci::Position posMax, bool scrolling) const;
 	void StartIdleStyling(bool truncatedLastStyling);
 	void StyleAreaBounded(PRectangle rcArea, bool scrolling);
+	constexpr bool SynchronousStylingToVisible() const noexcept {
+		return (idleStyling == SC_IDLESTYLING_NONE) || (idleStyling == SC_IDLESTYLING_AFTERVISIBLE);
+	}
 	void IdleStyling();
 	virtual void IdleWork();
 	virtual void QueueIdleWork(WorkNeeded::workItems items, Sci::Position upTo=0);
@@ -597,6 +599,8 @@ protected:	// ScintillaBase subclass needs access to much of Editor
 	static sptr_t BytesResult(sptr_t lParam, const unsigned char *val, size_t len) noexcept;
 
 public:
+	~Editor() override;
+
 	// Public so the COM thunks can access it.
 	bool IsUnicodeMode() const noexcept;
 	// Public so scintilla_send_message can use it.

scintilla/src/ElapsedPeriod.h

@@ -15,7 +15,7 @@ class ElapsedPeriod {
 	std::chrono::high_resolution_clock::time_point tp;
 public:
 	/// Capture the moment
-	ElapsedPeriod() : tp(std::chrono::high_resolution_clock::now()) {
+	ElapsedPeriod() noexcept : tp(std::chrono::high_resolution_clock::now()) {
 	}
 	/// Return duration as floating point seconds
 	double Duration(bool reset=false) {

scintilla/src/Indicator.cxx

@@ -23,10 +23,10 @@
 
 using namespace Scintilla;
 
-static PRectangle PixelGridAlign(const PRectangle &rc) {
+static PRectangle PixelGridAlign(const PRectangle &rc) noexcept {
 	// Move left and right side to nearest pixel to avoid blurry visuals
-	return PRectangle(round(rc.left), floor(rc.top),
+	return PRectangle(std::round(rc.left), std::floor(rc.top),
-		round(rc.right), floor(rc.bottom));
+		std::round(rc.right), std::floor(rc.bottom));
 }
 
 void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &rcLine, const PRectangle &rcCharacter, DrawState drawState, int value) const {
@@ -201,10 +201,10 @@ void Indicator::Draw(Surface *surface, const PRectangle &rc, const PRectangle &r
 		surface->FillRectangle(rcComposition, sacDraw.fore);
 	} else if (sacDraw.style == INDIC_POINT || sacDraw.style == INDIC_POINTCHARACTER) {
 		if (rcCharacter.Width() >= 0.1) {
-			const XYPOSITION pixelHeight = floor(rc.Height() - 1.0f);	// 1 pixel onto next line if multiphase
+			const XYPOSITION pixelHeight = std::floor(rc.Height() - 1.0f);	// 1 pixel onto next line if multiphase
 			const XYPOSITION x = (sacDraw.style == INDIC_POINT) ? (rcCharacter.left) : ((rcCharacter.right + rcCharacter.left) / 2);
-			const XYPOSITION ix = round(x);
+			const XYPOSITION ix = std::round(x);
-			const XYPOSITION iy = floor(rc.top + 1.0f);
+			const XYPOSITION iy = std::floor(rc.top + 1.0f);
 			Point pts[] = {
 				Point(ix - pixelHeight, iy + pixelHeight),	// Left
 				Point(ix + pixelHeight, iy + pixelHeight),	// Right

scintilla/src/Indicator.h

@@ -13,11 +13,11 @@ namespace Scintilla {
 struct StyleAndColour {
 	int style;
 	ColourDesired fore;
-	StyleAndColour() : style(INDIC_PLAIN), fore(0, 0, 0) {
+	StyleAndColour() noexcept : style(INDIC_PLAIN), fore(0, 0, 0) {
 	}
-	StyleAndColour(int style_, ColourDesired fore_ = ColourDesired(0, 0, 0)) : style(style_), fore(fore_) {
+	StyleAndColour(int style_, ColourDesired fore_ = ColourDesired(0, 0, 0)) noexcept : style(style_), fore(fore_) {
 	}
-	bool operator==(const StyleAndColour &other) const {
+	bool operator==(const StyleAndColour &other) const noexcept {
 		return (style == other.style) && (fore == other.fore);
 	}
 };
@@ -33,19 +33,19 @@ public:
 	int fillAlpha;
 	int outlineAlpha;
 	int attributes;
-	Indicator() : under(false), fillAlpha(30), outlineAlpha(50), attributes(0) {
+	Indicator() noexcept : under(false), fillAlpha(30), outlineAlpha(50), attributes(0) {
 	}
-	Indicator(int style_, ColourDesired fore_=ColourDesired(0,0,0), bool under_=false, int fillAlpha_=30, int outlineAlpha_=50) :
+	Indicator(int style_, ColourDesired fore_=ColourDesired(0,0,0), bool under_=false, int fillAlpha_=30, int outlineAlpha_=50) noexcept :
 		sacNormal(style_, fore_), sacHover(style_, fore_), under(under_), fillAlpha(fillAlpha_), outlineAlpha(outlineAlpha_), attributes(0) {
 	}
 	void Draw(Surface *surface, const PRectangle &rc, const PRectangle &rcLine, const PRectangle &rcCharacter, DrawState drawState, int value) const;
-	bool IsDynamic() const {
+	bool IsDynamic() const noexcept {
 		return !(sacNormal == sacHover);
 	}
-	bool OverridesTextFore() const {
+	bool OverridesTextFore() const noexcept {
 		return sacNormal.style == INDIC_TEXTFORE || sacHover.style == INDIC_TEXTFORE;
 	}
-	int Flags() const {
+	int Flags() const noexcept {
 		return attributes;
 	}
 	void SetFlags(int attributes_);

scintilla/src/KeyMap.cxx

@@ -33,7 +33,7 @@ KeyMap::~KeyMap() {
 	Clear();
 }
 
-void KeyMap::Clear() {
+void KeyMap::Clear() noexcept {
 	kmap.clear();
 }
 
@@ -46,7 +46,7 @@ unsigned int KeyMap::Find(int key, int modifiers) const {
 	return (it == kmap.end()) ? 0 : it->second;
 }
 
-const std::map<KeyModifiers, unsigned int> &KeyMap::GetKeyMap() const {
+const std::map<KeyModifiers, unsigned int> &KeyMap::GetKeyMap() const noexcept {
 	return kmap;
 }
 

scintilla/src/KeyMap.h

@@ -25,9 +25,9 @@ class KeyModifiers {
 public:
 	int key;
 	int modifiers;
-	KeyModifiers(int key_, int modifiers_) : key(key_), modifiers(modifiers_) {
+	KeyModifiers(int key_, int modifiers_) noexcept : key(key_), modifiers(modifiers_) {
 	}
-	bool operator<(const KeyModifiers &other) const {
+	bool operator<(const KeyModifiers &other) const noexcept {
 		if (key == other.key)
 			return modifiers < other.modifiers;
 		else
@@ -53,10 +53,10 @@ class KeyMap {
 public:
 	KeyMap();
 	~KeyMap();
-	void Clear();
+	void Clear() noexcept;
 	void AssignCmdKey(int key, int modifiers, unsigned int msg);
 	unsigned int Find(int key, int modifiers) const;	// 0 returned on failure
-	const std::map<KeyModifiers, unsigned int> &GetKeyMap() const;
+	const std::map<KeyModifiers, unsigned int> &GetKeyMap() const noexcept;
 };
 
 }

scintilla/src/LineMarker.cxx

@@ -26,41 +26,41 @@
 
 using namespace Scintilla;
 
-LineMarker::~LineMarker() {
+LineMarker::LineMarker(const LineMarker &other) {
-}
-
-LineMarker::LineMarker() {
-	markType = SC_MARK_CIRCLE;
-	fore = ColourDesired(0, 0, 0);
-	back = ColourDesired(0xff, 0xff, 0xff);
-	backSelected = ColourDesired(0xff, 0x00, 0x00);
-	alpha = SC_ALPHA_NOALPHA;
-	customDraw = nullptr;
-}
-
-LineMarker::LineMarker(const LineMarker &) {
 	// Defined to avoid pxpm and image being blindly copied, not as a complete copy constructor.
-	markType = SC_MARK_CIRCLE;
+	markType = other.markType;
-	fore = ColourDesired(0, 0, 0);
+	fore = other.fore;
-	back = ColourDesired(0xff, 0xff, 0xff);
+	back = other.back;
-	backSelected = ColourDesired(0xff, 0x00, 0x00);
+	backSelected = other.backSelected;
-	alpha = SC_ALPHA_NOALPHA;
+	alpha = other.alpha;
-	pxpm.reset();
+	if (other.pxpm)
-	image.reset();
+		pxpm = std::make_unique<XPM>(*other.pxpm);
-	customDraw = nullptr;
+	else
+		pxpm = nullptr;
+	if (other.image)
+		image = std::make_unique<RGBAImage>(*other.image);
+	else
+		image = nullptr;
+	customDraw = other.customDraw;
 }
 
 LineMarker &LineMarker::operator=(const LineMarker &other) {
 	// Defined to avoid pxpm and image being blindly copied, not as a complete assignment operator.
 	if (this != &other) {
-		markType = SC_MARK_CIRCLE;
+		markType = other.markType;
-		fore = ColourDesired(0, 0, 0);
+		fore = other.fore;
-		back = ColourDesired(0xff, 0xff, 0xff);
+		back = other.back;
-		backSelected = ColourDesired(0xff, 0x00, 0x00);
+		backSelected = other.backSelected;
-		alpha = SC_ALPHA_NOALPHA;
+		alpha = other.alpha;
-		pxpm.reset();
+		if (other.pxpm)
-		image.reset();
+			pxpm = std::make_unique<XPM>(*other.pxpm);
-		customDraw = nullptr;
+		else
+			pxpm = nullptr;
+		if (other.image)
+			image = std::make_unique<RGBAImage>(*other.image);
+		else
+			image = nullptr;
+		customDraw = other.customDraw;
 	}
 	return *this;
 }
@@ -430,6 +430,16 @@ void LineMarker::Draw(Surface *surface, PRectangle &rcWhole, Font &fontForCharac
 			Point::FromInts(ircWhole.left, centreY + halfHeight),
 		};
 		surface->Polygon(pts, std::size(pts), fore, back);
+	} else if (markType == SC_MARK_VERTICALBOOKMARK) {
+		const int halfWidth = minDim / 3;
+		Point pts[] = {
+			Point::FromInts(centreX - halfWidth, centreY - dimOn2),
+			Point::FromInts(centreX + halfWidth, centreY - dimOn2),
+			Point::FromInts(centreX + halfWidth, centreY + dimOn2),
+			Point::FromInts(centreX, centreY + dimOn2 - halfWidth),
+			Point::FromInts(centreX - halfWidth, centreY + dimOn2),
+		};
+		surface->Polygon(pts, std::size(pts), fore, back);
 	} else { // SC_MARK_FULLRECT
 		surface->FillRectangle(rcWhole, back);
 	}

scintilla/src/LineMarker.h

@@ -21,22 +21,26 @@ class LineMarker {
 public:
 	enum typeOfFold { undefined, head, body, tail, headWithTail };
 
-	int markType;
+	int markType = SC_MARK_CIRCLE;
-	ColourDesired fore;
+	ColourDesired fore = ColourDesired(0, 0, 0);
-	ColourDesired back;
+	ColourDesired back = ColourDesired(0xff, 0xff, 0xff);
-	ColourDesired backSelected;
+	ColourDesired backSelected = ColourDesired(0xff, 0x00, 0x00);
-	int alpha;
+	int alpha = SC_ALPHA_NOALPHA;
 	std::unique_ptr<XPM> pxpm;
 	std::unique_ptr<RGBAImage> image;
 	/** Some platforms, notably PLAT_CURSES, do not support Scintilla's native
 	 * Draw function for drawing line markers. Allow those platforms to override
 	 * it instead of creating a new method(s) in the Surface class that existing
 	 * platforms must implement as empty. */
-	DrawLineMarkerFn customDraw;
+	DrawLineMarkerFn customDraw = nullptr;
-	LineMarker();
+
-	LineMarker(const LineMarker &);
+	LineMarker() noexcept = default;
-	virtual ~LineMarker();
+	LineMarker(const LineMarker &other);
-	LineMarker &operator=(const LineMarker &other);
+	LineMarker(LineMarker &&) noexcept = default;
+	LineMarker &operator=(const LineMarker& other);
+	LineMarker &operator=(LineMarker&&) noexcept = default;
+	virtual ~LineMarker() = default;
+
 	void SetXPM(const char *textForm);
 	void SetXPM(const char *const *linesForm);
 	void SetRGBAImage(Point sizeRGBAImage, float scale, const unsigned char *pixelsRGBAImage);

scintilla/src/MarginView.cxx

@@ -26,6 +26,7 @@
 #include "ILexer.h"
 #include "Scintilla.h"
 
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "IntegerRectangle.h"
 #include "UniqueString.h"
@@ -419,7 +420,7 @@ void MarginView::PaintMargin(Surface *surface, Sci::Line topLine, PRectangle rc,
 							DrawStyledText(surface, vs, vs.marginStyleOffset, rcMarker,
 								stMargin, 0, stMargin.length, drawAll);
 						} else {
-							// if we're displaying annotation lines, color the margin to match the associated document line
+							// if we're displaying annotation lines, colour the margin to match the associated document line
 							const int annotationLines = model.pdoc->AnnotationLines(lineDoc);
 							if (annotationLines && (visibleLine > lastVisibleLine - annotationLines)) {
 								surface->FillRectangle(rcMarker, vs.styles[stMargin.StyleAt(0) + vs.marginStyleOffset].back);

scintilla/src/PerLine.h

@@ -17,7 +17,7 @@ namespace Scintilla {
 struct MarkerHandleNumber {
 	int handle;
 	int number;
-	MarkerHandleNumber(int handle_, int number_) : handle(handle_), number(number_) {}
+	MarkerHandleNumber(int handle_, int number_) noexcept : handle(handle_), number(number_) {}
 };
 
 /**

scintilla/src/PositionCache.cxx

@@ -25,6 +25,7 @@
 #include "ILexer.h"
 #include "Scintilla.h"
 
+#include "CharacterCategory.h"
 #include "Position.h"
 #include "UniqueString.h"
 #include "SplitVector.h"
@@ -348,7 +349,7 @@ XYPOSITION ScreenLine::RepresentationWidth(size_t position) const {
 }
 
 XYPOSITION ScreenLine::TabPositionAfter(XYPOSITION xPosition) const {
-	return (floor((xPosition + TabWidthMinimumPixels()) / TabWidth()) + 1) * TabWidth();
+	return (std::floor((xPosition + TabWidthMinimumPixels()) / TabWidth()) + 1) * TabWidth();
 }
 
 LineLayoutCache::LineLayoutCache() :
@@ -793,13 +794,13 @@ void PositionCache::MeasureWidths(Surface *surface, const ViewStyle &vstyle, uns
 			probe = probe2;
 		}
 	}
+	FontAlias fontStyle = vstyle.styles[styleNumber].font;
 	if (len > BreakFinder::lengthStartSubdivision) {
 		// Break up into segments
 		unsigned int startSegment = 0;
 		XYPOSITION xStartSegment = 0;
 		while (startSegment < len) {
 			const unsigned int lenSegment = pdoc->SafeSegment(s + startSegment, len - startSegment, BreakFinder::lengthEachSubdivision);
-			FontAlias fontStyle = vstyle.styles[styleNumber].font;
 			surface->MeasureWidths(fontStyle, std::string_view(s + startSegment, lenSegment), positions + startSegment);
 			for (unsigned int inSeg = 0; inSeg < lenSegment; inSeg++) {
 				positions[startSegment + inSeg] += xStartSegment;
@@ -808,7 +809,6 @@ void PositionCache::MeasureWidths(Surface *surface, const ViewStyle &vstyle, uns
 			startSegment += lenSegment;
 		}
 	} else {
-		FontAlias fontStyle = vstyle.styles[styleNumber].font;
 		surface->MeasureWidths(fontStyle, std::string_view(s, len), positions);
 	}
 	if (probe < pces.size()) {

scintilla/src/RESearch.h

@@ -11,14 +11,6 @@
 
 namespace Scintilla {
 
-/*
- * The following defines are not meant to be changeable.
- * They are for readability only.
- */
-#define MAXCHR	256
-#define CHRBIT	8
-#define BITBLK	MAXCHR/CHRBIT
-
 class CharacterIndexer {
 public:
 	virtual char CharAt(Sci::Position index) const=0;
@@ -38,7 +30,6 @@ public:
 	int Execute(const CharacterIndexer &ci, Sci::Position lp, Sci::Position endp);
 
 	enum { MAXTAG=10 };
-	enum { MAXNFA=4096 };
 	enum { NOTFOUND=-1 };
 
 	Sci::Position bopat[MAXTAG];
@@ -46,6 +37,14 @@ public:
 	std::string pat[MAXTAG];
 
 private:
+
+	enum { MAXNFA = 4096 };
+	// The following enums are not meant to be changeable.
+	// They are for readability only.
+	enum { MAXCHR = 256 };
+	enum { CHRBIT = 8 };
+	enum { BITBLK = MAXCHR / CHRBIT };
+
 	void ChSet(unsigned char c);
 	void ChSetWithCase(unsigned char c, bool caseSensitive);
 	int GetBackslashExpression(const char *pattern, int &incr);

scintilla/src/ScintillaBase.cxx

@@ -29,6 +29,7 @@
 #endif
 
 #include "PropSetSimple.h"
+#include "CharacterCategory.h"
 
 #ifdef SCI_LEXER
 #include "LexerModule.h"
@@ -78,17 +79,21 @@ void ScintillaBase::Finalise() {
 	popup.Destroy();
 }
 
-void ScintillaBase::AddCharUTF(const char *s, unsigned int len, bool treatAsDBCS) {
+void ScintillaBase::AddCharUTF(const char *s, unsigned int len, bool /*treatAsDBCS*/) {
-	const bool isFillUp = ac.Active() && ac.IsFillUpChar(*s);
+	InsertCharacter(std::string_view(s, len), CharacterSource::directInput);
+}
+
+void ScintillaBase::InsertCharacter(std::string_view sv, CharacterSource charSource) {
+	const bool isFillUp = ac.Active() && ac.IsFillUpChar(sv[0]);
 	if (!isFillUp) {
-		Editor::AddCharUTF(s, len, treatAsDBCS);
+		Editor::InsertCharacter(sv, charSource);
 	}
 	if (ac.Active()) {
-		AutoCompleteCharacterAdded(s[0]);
+		AutoCompleteCharacterAdded(sv[0]);
 		// For fill ups add the character after the autocompletion has
 		// triggered so containers see the key so can display a calltip.
 		if (isFillUp) {
-			Editor::AddCharUTF(s, len, treatAsDBCS);
+			Editor::InsertCharacter(sv, charSource);
 		}
 	}
 }
@@ -276,9 +281,7 @@ void ScintillaBase::AutoCompleteStart(Sci::Position lenEntered, const char *list
 		pt = PointMainCaret();
 	}
 	if (wMargin.Created()) {
-		const Point ptOrigin = GetVisibleOriginInMain();
+		pt = pt + GetVisibleOriginInMain();
-		pt.x += ptOrigin.x;
-		pt.y += ptOrigin.y;
 	}
 	PRectangle rcac;
 	rcac.left = pt.x - ac.lb->CaretFromEdge();
@@ -463,9 +466,7 @@ void ScintillaBase::CallTipShow(Point pt, const char *defn) {
 		ct.SetForeBack(vs.styles[STYLE_CALLTIP].fore, vs.styles[STYLE_CALLTIP].back);
 	}
 	if (wMargin.Created()) {
-		const Point ptOrigin = GetVisibleOriginInMain();
+		pt = pt + GetVisibleOriginInMain();
-		pt.x += ptOrigin.x;
-		pt.y += ptOrigin.y;
 	}
 	PRectangle rc = ct.CallTipStart(sel.MainCaret(), pt,
 		vs.lineHeight,

scintilla/src/ScintillaBase.h

@@ -32,8 +32,6 @@ protected:
 		idcmdSelectAll=16
 	};
 
-	enum { maxLenInputIME = 200 };
-
 	int displayPopupMenu;
 	Menu popup;
 	AutoComplete ac;
@@ -57,11 +55,15 @@ protected:
 	ScintillaBase(ScintillaBase &&) = delete;
 	ScintillaBase &operator=(const ScintillaBase &) = delete;
 	ScintillaBase &operator=(ScintillaBase &&) = delete;
-	~ScintillaBase() override;
+	// ~ScintillaBase() in public section
 	void Initialise() override {}
 	void Finalise() override;
 
-	void AddCharUTF(const char *s, unsigned int len, bool treatAsDBCS=false) override;
+	[[deprecated]]
+	// This method is deprecated, use InsertCharacter instead. The treatAsDBCS parameter is no longer used.
+	virtual void AddCharUTF(const char *s, unsigned int len, bool treatAsDBCS=false);
+
+	void InsertCharacter(std::string_view sv, CharacterSource charSource) override;
 	void Command(int cmdId);
 	void CancelModes() override;
 	int KeyCommand(unsigned int iMessage) override;
@@ -94,6 +96,8 @@ protected:
 	void NotifyLexerChanged(Document *doc, void *userData) override;
 
 public:
+	~ScintillaBase() override;
+
 	// Public so scintilla_send_message can use it
 	sptr_t WndProc(unsigned int iMessage, uptr_t wParam, sptr_t lParam) override;
 };

scintilla/src/Selection.cxx

@@ -23,7 +23,7 @@
 
 using namespace Scintilla;
 
-void SelectionPosition::MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length) {
+void SelectionPosition::MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length) noexcept {
 	if (insertion) {
 		if (position == startChange) {
 			const Sci::Position virtualLengthRemove = std::min(length, virtualSpace);
@@ -48,35 +48,35 @@ void SelectionPosition::MoveForInsertDelete(bool insertion, Sci::Position startC
 	}
 }
 
-bool SelectionPosition::operator <(const SelectionPosition &other) const {
+bool SelectionPosition::operator <(const SelectionPosition &other) const noexcept {
 	if (position == other.position)
 		return virtualSpace < other.virtualSpace;
 	else
 		return position < other.position;
 }
 
-bool SelectionPosition::operator >(const SelectionPosition &other) const {
+bool SelectionPosition::operator >(const SelectionPosition &other) const noexcept {
 	if (position == other.position)
 		return virtualSpace > other.virtualSpace;
 	else
 		return position > other.position;
 }
 
-bool SelectionPosition::operator <=(const SelectionPosition &other) const {
+bool SelectionPosition::operator <=(const SelectionPosition &other) const noexcept {
 	if (position == other.position && virtualSpace == other.virtualSpace)
 		return true;
 	else
 		return other > *this;
 }
 
-bool SelectionPosition::operator >=(const SelectionPosition &other) const {
+bool SelectionPosition::operator >=(const SelectionPosition &other) const noexcept {
 	if (position == other.position && virtualSpace == other.virtualSpace)
 		return true;
 	else
 		return *this > other;
 }
 
-Sci::Position SelectionRange::Length() const {
+Sci::Position SelectionRange::Length() const noexcept {
 	if (anchor > caret) {
 		return anchor.Position() - caret.Position();
 	} else {
@@ -84,33 +84,33 @@ Sci::Position SelectionRange::Length() const {
 	}
 }
 
-void SelectionRange::MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length) {
+void SelectionRange::MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length) noexcept {
 	caret.MoveForInsertDelete(insertion, startChange, length);
 	anchor.MoveForInsertDelete(insertion, startChange, length);
 }
 
-bool SelectionRange::Contains(Sci::Position pos) const {
+bool SelectionRange::Contains(Sci::Position pos) const noexcept {
 	if (anchor > caret)
 		return (pos >= caret.Position()) && (pos <= anchor.Position());
 	else
 		return (pos >= anchor.Position()) && (pos <= caret.Position());
 }
 
-bool SelectionRange::Contains(SelectionPosition sp) const {
+bool SelectionRange::Contains(SelectionPosition sp) const noexcept {
 	if (anchor > caret)
 		return (sp >= caret) && (sp <= anchor);
 	else
 		return (sp >= anchor) && (sp <= caret);
 }
 
-bool SelectionRange::ContainsCharacter(Sci::Position posCharacter) const {
+bool SelectionRange::ContainsCharacter(Sci::Position posCharacter) const noexcept {
 	if (anchor > caret)
 		return (posCharacter >= caret.Position()) && (posCharacter < anchor.Position());
 	else
 		return (posCharacter >= anchor.Position()) && (posCharacter < caret.Position());
 }
 
-SelectionSegment SelectionRange::Intersect(SelectionSegment check) const {
+SelectionSegment SelectionRange::Intersect(SelectionSegment check) const noexcept {
 	const SelectionSegment inOrder(caret, anchor);
 	if ((inOrder.start <= check.end) || (inOrder.end >= check.start)) {
 		SelectionSegment portion = check;
@@ -127,11 +127,11 @@ SelectionSegment SelectionRange::Intersect(SelectionSegment check) const {
 	}
 }
 
-void SelectionRange::Swap() {
+void SelectionRange::Swap() noexcept {
 	std::swap(caret, anchor);
 }
 
-bool SelectionRange::Trim(SelectionRange range) {
+bool SelectionRange::Trim(SelectionRange range) noexcept {
 	const SelectionPosition startRange = range.Start();
 	const SelectionPosition endRange = range.End();
 	SelectionPosition start = Start();
@@ -167,7 +167,7 @@ bool SelectionRange::Trim(SelectionRange range) {
 }
 
 // If range is all virtual collapse to start of virtual space
-void SelectionRange::MinimizeVirtualSpace() {
+void SelectionRange::MinimizeVirtualSpace() noexcept {
 	if (caret.Position() == anchor.Position()) {
 		Sci::Position virtualSpace = caret.VirtualSpace();
 		if (virtualSpace > anchor.VirtualSpace())
@@ -184,7 +184,7 @@ Selection::Selection() : mainRange(0), moveExtends(false), tentativeMain(false),
 Selection::~Selection() {
 }
 
-bool Selection::IsRectangular() const {
+bool Selection::IsRectangular() const noexcept {
 	return (selType == selRectangle) || (selType == selThin);
 }
 
@@ -196,7 +196,7 @@ Sci::Position Selection::MainAnchor() const {
 	return ranges[mainRange].anchor.Position();
 }
 
-SelectionRange &Selection::Rectangular() {
+SelectionRange &Selection::Rectangular() noexcept {
 	return rangeRectangular;
 }
 
@@ -221,15 +221,15 @@ SelectionSegment Selection::LimitsForRectangularElseMain() const {
 	}
 }
 
-size_t Selection::Count() const {
+size_t Selection::Count() const noexcept {
 	return ranges.size();
 }
 
-size_t Selection::Main() const {
+size_t Selection::Main() const noexcept {
 	return mainRange;
 }
 
-void Selection::SetMain(size_t r) {
+void Selection::SetMain(size_t r) noexcept {
 	PLATFORM_ASSERT(r < ranges.size());
 	mainRange = r;
 }
@@ -258,15 +258,15 @@ SelectionPosition Selection::Start() const {
 	}
 }
 
-bool Selection::MoveExtends() const {
+bool Selection::MoveExtends() const noexcept {
 	return moveExtends;
 }
 
-void Selection::SetMoveExtends(bool moveExtends_) {
+void Selection::SetMoveExtends(bool moveExtends_) noexcept {
 	moveExtends = moveExtends_;
 }
 
-bool Selection::Empty() const {
+bool Selection::Empty() const noexcept {
 	for (const SelectionRange &range : ranges) {
 		if (!range.Empty())
 			return false;
@@ -274,7 +274,7 @@ bool Selection::Empty() const {
 	return true;
 }
 
-SelectionPosition Selection::Last() const {
+SelectionPosition Selection::Last() const noexcept {
 	SelectionPosition lastPosition;
 	for (const SelectionRange &range : ranges) {
 		if (lastPosition < range.caret)
@@ -285,7 +285,7 @@ SelectionPosition Selection::Last() const {
 	return lastPosition;
 }
 
-Sci::Position Selection::Length() const {
+Sci::Position Selection::Length() const noexcept {
 	Sci::Position len = 0;
 	for (const SelectionRange &range : ranges) {
 		len += range.Length();
@@ -293,7 +293,7 @@ Sci::Position Selection::Length() const {
 	return len;
 }
 
-void Selection::MovePositions(bool insertion, Sci::Position startChange, Sci::Position length) {
+void Selection::MovePositions(bool insertion, Sci::Position startChange, Sci::Position length) noexcept {
 	for (SelectionRange &range : ranges) {
 		range.MoveForInsertDelete(insertion, startChange, length);
 	}
@@ -372,7 +372,7 @@ void Selection::TentativeSelection(SelectionRange range) {
 	tentativeMain = true;
 }
 
-void Selection::CommitTentative() {
+void Selection::CommitTentative() noexcept {
 	rangesSaved.clear();
 	tentativeMain = false;
 }
@@ -393,7 +393,7 @@ int Selection::InSelectionForEOL(Sci::Position pos) const {
 	return 0;
 }
 
-Sci::Position Selection::VirtualSpaceFor(Sci::Position pos) const {
+Sci::Position Selection::VirtualSpaceFor(Sci::Position pos) const noexcept {
 	Sci::Position virtualSpace = 0;
 	for (const SelectionRange &range : ranges) {
 		if ((range.caret.Position() == pos) && (virtualSpace < range.caret.VirtualSpace()))
@@ -431,7 +431,7 @@ void Selection::RemoveDuplicates() {
 	}
 }
 
-void Selection::RotateMain() {
+void Selection::RotateMain() noexcept {
 	mainRange = (mainRange + 1) % ranges.size();
 }
 

scintilla/src/Selection.h

@@ -14,42 +14,42 @@ class SelectionPosition {
 	Sci::Position position;
 	Sci::Position virtualSpace;
 public:
-	explicit SelectionPosition(Sci::Position position_=INVALID_POSITION, Sci::Position virtualSpace_=0) : position(position_), virtualSpace(virtualSpace_) {
+	explicit SelectionPosition(Sci::Position position_=INVALID_POSITION, Sci::Position virtualSpace_=0) noexcept : position(position_), virtualSpace(virtualSpace_) {
 		PLATFORM_ASSERT(virtualSpace < 800000);
 		if (virtualSpace < 0)
 			virtualSpace = 0;
 	}
-	void Reset() {
+	void Reset() noexcept {
 		position = 0;
 		virtualSpace = 0;
 	}
-	void MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length);
+	void MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length) noexcept;
-	bool operator ==(const SelectionPosition &other) const {
+	bool operator ==(const SelectionPosition &other) const noexcept {
 		return position == other.position && virtualSpace == other.virtualSpace;
 	}
-	bool operator <(const SelectionPosition &other) const;
+	bool operator <(const SelectionPosition &other) const noexcept;
-	bool operator >(const SelectionPosition &other) const;
+	bool operator >(const SelectionPosition &other) const noexcept;
-	bool operator <=(const SelectionPosition &other) const;
+	bool operator <=(const SelectionPosition &other) const noexcept;
-	bool operator >=(const SelectionPosition &other) const;
+	bool operator >=(const SelectionPosition &other) const noexcept;
-	Sci::Position Position() const {
+	Sci::Position Position() const noexcept {
 		return position;
 	}
-	void SetPosition(Sci::Position position_) {
+	void SetPosition(Sci::Position position_) noexcept {
 		position = position_;
 		virtualSpace = 0;
 	}
-	Sci::Position VirtualSpace() const {
+	Sci::Position VirtualSpace() const noexcept {
 		return virtualSpace;
 	}
-	void SetVirtualSpace(Sci::Position virtualSpace_) {
+	void SetVirtualSpace(Sci::Position virtualSpace_) noexcept {
 		PLATFORM_ASSERT(virtualSpace_ < 800000);
 		if (virtualSpace_ >= 0)
 			virtualSpace = virtualSpace_;
 	}
-	void Add(Sci::Position increment) {
+	void Add(Sci::Position increment) noexcept {
 		position = position + increment;
 	}
-	bool IsValid() const {
+	bool IsValid() const noexcept {
 		return position >= 0;
 	}
 };
@@ -58,9 +58,9 @@ public:
 struct SelectionSegment {
 	SelectionPosition start;
 	SelectionPosition end;
-	SelectionSegment() : start(), end() {
+	SelectionSegment() noexcept : start(), end() {
 	}
-	SelectionSegment(SelectionPosition a, SelectionPosition b) {
+	SelectionSegment(SelectionPosition a, SelectionPosition b) noexcept {
 		if (a < b) {
 			start = a;
 			end = b;
@@ -69,10 +69,10 @@ struct SelectionSegment {
 			end = a;
 		}
 	}
-	bool Empty() const {
+	bool Empty() const noexcept {
 		return start == end;
 	}
-	void Extend(SelectionPosition p) {
+	void Extend(SelectionPosition p) noexcept {
 		if (start > p)
 			start = p;
 		if (end < p)
@@ -84,50 +84,50 @@ struct SelectionRange {
 	SelectionPosition caret;
 	SelectionPosition anchor;
 
-	SelectionRange() : caret(), anchor() {
+	SelectionRange() noexcept : caret(), anchor() {
 	}
-	explicit SelectionRange(SelectionPosition single) : caret(single), anchor(single) {
+	explicit SelectionRange(SelectionPosition single) noexcept : caret(single), anchor(single) {
 	}
-	explicit SelectionRange(Sci::Position single) : caret(single), anchor(single) {
+	explicit SelectionRange(Sci::Position single) noexcept : caret(single), anchor(single) {
 	}
-	SelectionRange(SelectionPosition caret_, SelectionPosition anchor_) : caret(caret_), anchor(anchor_) {
+	SelectionRange(SelectionPosition caret_, SelectionPosition anchor_) noexcept : caret(caret_), anchor(anchor_) {
 	}
-	SelectionRange(Sci::Position caret_, Sci::Position anchor_) : caret(caret_), anchor(anchor_) {
+	SelectionRange(Sci::Position caret_, Sci::Position anchor_) noexcept : caret(caret_), anchor(anchor_) {
 	}
-	bool Empty() const {
+	bool Empty() const noexcept {
 		return anchor == caret;
 	}
-	Sci::Position Length() const;
+	Sci::Position Length() const noexcept;
 	// Sci::Position Width() const;	// Like Length but takes virtual space into account
-	bool operator ==(const SelectionRange &other) const {
+	bool operator ==(const SelectionRange &other) const noexcept {
 		return caret == other.caret && anchor == other.anchor;
 	}
-	bool operator <(const SelectionRange &other) const {
+	bool operator <(const SelectionRange &other) const noexcept {
 		return caret < other.caret || ((caret == other.caret) && (anchor < other.anchor));
 	}
-	void Reset() {
+	void Reset() noexcept {
 		anchor.Reset();
 		caret.Reset();
 	}
-	void ClearVirtualSpace() {
+	void ClearVirtualSpace() noexcept {
 		anchor.SetVirtualSpace(0);
 		caret.SetVirtualSpace(0);
 	}
-	void MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length);
+	void MoveForInsertDelete(bool insertion, Sci::Position startChange, Sci::Position length) noexcept;
-	bool Contains(Sci::Position pos) const;
+	bool Contains(Sci::Position pos) const noexcept;
-	bool Contains(SelectionPosition sp) const;
+	bool Contains(SelectionPosition sp) const noexcept;
-	bool ContainsCharacter(Sci::Position posCharacter) const;
+	bool ContainsCharacter(Sci::Position posCharacter) const noexcept;
-	SelectionSegment Intersect(SelectionSegment check) const;
+	SelectionSegment Intersect(SelectionSegment check) const noexcept;
-	SelectionPosition Start() const {
+	SelectionPosition Start() const noexcept {
 		return (anchor < caret) ? anchor : caret;
 	}
-	SelectionPosition End() const {
+	SelectionPosition End() const noexcept {
 		return (anchor < caret) ? caret : anchor;
 	}
-	void Swap();
+	void Swap() noexcept;
-	bool Trim(SelectionRange range);
+	bool Trim(SelectionRange range) noexcept;
 	// If range is all virtual collapse to start of virtual space
-	void MinimizeVirtualSpace();
+	void MinimizeVirtualSpace() noexcept;
 };
 
 class Selection {
@@ -143,29 +143,29 @@ public:
 
 	Selection();
 	~Selection();
-	bool IsRectangular() const;
+	bool IsRectangular() const noexcept;
 	Sci::Position MainCaret() const;
 	Sci::Position MainAnchor() const;
-	SelectionRange &Rectangular();
+	SelectionRange &Rectangular() noexcept;
 	SelectionSegment Limits() const;
 	// This is for when you want to move the caret in response to a
 	// user direction command - for rectangular selections, use the range
 	// that covers all selected text otherwise return the main selection.
 	SelectionSegment LimitsForRectangularElseMain() const;
-	size_t Count() const;
+	size_t Count() const noexcept;
-	size_t Main() const;
+	size_t Main() const noexcept;
-	void SetMain(size_t r);
+	void SetMain(size_t r) noexcept;
 	SelectionRange &Range(size_t r);
 	const SelectionRange &Range(size_t r) const;
 	SelectionRange &RangeMain();
 	const SelectionRange &RangeMain() const;
 	SelectionPosition Start() const;
-	bool MoveExtends() const;
+	bool MoveExtends() const noexcept;
-	void SetMoveExtends(bool moveExtends_);
+	void SetMoveExtends(bool moveExtends_) noexcept;
-	bool Empty() const;
+	bool Empty() const noexcept;
-	SelectionPosition Last() const;
+	SelectionPosition Last() const noexcept;
-	Sci::Position Length() const;
+	Sci::Position Length() const noexcept;
-	void MovePositions(bool insertion, Sci::Position startChange, Sci::Position length);
+	void MovePositions(bool insertion, Sci::Position startChange, Sci::Position length) noexcept;
 	void TrimSelection(SelectionRange range);
 	void TrimOtherSelections(size_t r, SelectionRange range);
 	void SetSelection(SelectionRange range);
@@ -174,14 +174,14 @@ public:
 	void DropSelection(size_t r);
 	void DropAdditionalRanges();
 	void TentativeSelection(SelectionRange range);
-	void CommitTentative();
+	void CommitTentative() noexcept;
 	int CharacterInSelection(Sci::Position posCharacter) const;
 	int InSelectionForEOL(Sci::Position pos) const;
-	Sci::Position VirtualSpaceFor(Sci::Position pos) const;
+	Sci::Position VirtualSpaceFor(Sci::Position pos) const noexcept;
 	void Clear();
 	void RemoveDuplicates();
-	void RotateMain();
+	void RotateMain() noexcept;
-	bool Tentative() const { return tentativeMain; }
+	bool Tentative() const noexcept { return tentativeMain; }
 	std::vector<SelectionRange> RangesCopy() const {
 		return ranges;
 	}

scintilla/src/UniConversion.cxx

@@ -17,7 +17,7 @@ using namespace Scintilla;
 
 namespace Scintilla {
 
-size_t UTF8Length(std::wstring_view wsv) {
+size_t UTF8Length(std::wstring_view wsv) noexcept {
 	size_t len = 0;
 	for (size_t i = 0; i < wsv.length() && wsv[i];) {
 		const unsigned int uch = wsv[i];
@@ -78,7 +78,7 @@ void UTF8FromUTF16(std::wstring_view wsv, char *putf, size_t len) {
 		putf[k] = '\0';
 }
 
-void UTF8FromUTF32Character(int uch, char *putf) {
+void UTF8FromUTF32Character(int uch, char *putf) noexcept {
 	size_t k = 0;
 	if (uch < 0x80) {
 		putf[k++] = static_cast<char>(uch);
@@ -98,14 +98,14 @@ void UTF8FromUTF32Character(int uch, char *putf) {
 	putf[k] = '\0';
 }
 
-size_t UTF16Length(std::string_view sv) {
+size_t UTF16Length(std::string_view svu8) noexcept {
 	size_t ulen = 0;
-	for (size_t i = 0; i<sv.length();) {
+	for (size_t i = 0; i< svu8.length();) {
-		const unsigned char ch = sv[i];
+		const unsigned char ch = svu8[i];
 		const unsigned int byteCount = UTF8BytesOfLead[ch];
 		const unsigned int utf16Len = UTF16LengthFromUTF8ByteCount(byteCount);
 		i += byteCount;
-		ulen += (i > sv.length()) ? 1 : utf16Len;
+		ulen += (i > svu8.length()) ? 1 : utf16Len;
 	}
 	return ulen;
 }
@@ -116,14 +116,14 @@ constexpr unsigned char TrailByteValue(unsigned char c) {
 	return c & 0b0011'1111;
 }
 
-size_t UTF16FromUTF8(std::string_view sv, wchar_t *tbuf, size_t tlen) {
+size_t UTF16FromUTF8(std::string_view svu8, wchar_t *tbuf, size_t tlen) {
 	size_t ui = 0;
-	for (size_t i = 0; i < sv.length();) {
+	for (size_t i = 0; i < svu8.length();) {
-		unsigned char ch = sv[i];
+		unsigned char ch = svu8[i];
 		const unsigned int byteCount = UTF8BytesOfLead[ch];
 		unsigned int value;
 
-		if (i + byteCount > sv.length()) {
+		if (i + byteCount > svu8.length()) {
 			// Trying to read past end but still have space to write
 			if (ui < tlen) {
 				tbuf[ui] = ch;
@@ -144,26 +144,26 @@ size_t UTF16FromUTF8(std::string_view sv, wchar_t *tbuf, size_t tlen) {
 			break;
 		case 2:
 			value = (ch & 0x1F) << 6;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch);
 			tbuf[ui] = static_cast<wchar_t>(value);
 			break;
 		case 3:
 			value = (ch & 0xF) << 12;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += (TrailByteValue(ch) << 6);
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch);
 			tbuf[ui] = static_cast<wchar_t>(value);
 			break;
 		default:
 			// Outside the BMP so need two surrogates
 			value = (ch & 0x7) << 18;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch) << 12;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch) << 6;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch);
 			tbuf[ui] = static_cast<wchar_t>(((value - 0x10000) >> 10) + SURROGATE_LEAD_FIRST);
 			ui++;
@@ -175,14 +175,25 @@ size_t UTF16FromUTF8(std::string_view sv, wchar_t *tbuf, size_t tlen) {
 	return ui;
 }
 
-size_t UTF32FromUTF8(std::string_view sv, unsigned int *tbuf, size_t tlen) {
+size_t UTF32Length(std::string_view svu8) noexcept {
+	size_t ulen = 0;
+	for (size_t i = 0; i < svu8.length();) {
+		const unsigned char ch = svu8[i];
+		const unsigned int byteCount = UTF8BytesOfLead[ch];
+		i += byteCount;
+		ulen++;
+	}
+	return ulen;
+}
+
+size_t UTF32FromUTF8(std::string_view svu8, unsigned int *tbuf, size_t tlen) {
 	size_t ui = 0;
-	for (size_t i = 0; i < sv.length();) {
+	for (size_t i = 0; i < svu8.length();) {
-		unsigned char ch = sv[i];
+		unsigned char ch = svu8[i];
 		const unsigned int byteCount = UTF8BytesOfLead[ch];
 		unsigned int value;
 
-		if (i + byteCount > sv.length()) {
+		if (i + byteCount > svu8.length()) {
 			// Trying to read past end but still have space to write
 			if (ui < tlen) {
 				tbuf[ui] = ch;
@@ -202,23 +213,23 @@ size_t UTF32FromUTF8(std::string_view sv, unsigned int *tbuf, size_t tlen) {
 			break;
 		case 2:
 			value = (ch & 0x1F) << 6;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch);
 			break;
 		case 3:
 			value = (ch & 0xF) << 12;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch) << 6;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch);
 			break;
 		default:
 			value = (ch & 0x7) << 18;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch) << 12;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch) << 6;
-			ch = sv[i++];
+			ch = svu8[i++];
 			value += TrailByteValue(ch);
 			break;
 		}
@@ -228,6 +239,20 @@ size_t UTF32FromUTF8(std::string_view sv, unsigned int *tbuf, size_t tlen) {
 	return ui;
 }
 
+std::wstring WStringFromUTF8(std::string_view svu8) {
+	if constexpr (sizeof(wchar_t) == 2) {
+		const size_t len16 = UTF16Length(svu8);
+		std::wstring ws(len16, 0);
+		UTF16FromUTF8(svu8, &ws[0], len16);
+		return ws;
+	} else {
+		const size_t len32 = UTF32Length(svu8);
+		std::wstring ws(len32, 0);
+		UTF32FromUTF8(svu8, reinterpret_cast<unsigned int *>(&ws[0]), len32);
+		return ws;
+	}
+}
+
 unsigned int UTF16FromUTF32Character(unsigned int val, wchar_t *tbuf) noexcept {
 	if (val < SUPPLEMENTAL_PLANE_FIRST) {
 		tbuf[0] = static_cast<wchar_t>(val);
@@ -340,9 +365,9 @@ int UTF8DrawBytes(const unsigned char *us, int len) noexcept {
 	return (utf8StatusNext & UTF8MaskInvalid) ? 1 : (utf8StatusNext & UTF8MaskWidth);
 }
 
-bool UTF8IsValid(std::string_view sv) noexcept {
+bool UTF8IsValid(std::string_view svu8) noexcept {
-	const unsigned char *us = reinterpret_cast<const unsigned char *>(sv.data());
+	const unsigned char *us = reinterpret_cast<const unsigned char *>(svu8.data());
-	size_t remaining = sv.length();
+	size_t remaining = svu8.length();
 	while (remaining > 0) {
 		const int utf8Status = UTF8Classify(us, remaining);
 		if (utf8Status & UTF8MaskInvalid) {

scintilla/src/UniConversion.h

@@ -14,15 +14,19 @@ const int UTF8MaxBytes = 4;
 
 const int unicodeReplacementChar = 0xFFFD;
 
-size_t UTF8Length(std::wstring_view wsv);
+size_t UTF8Length(std::wstring_view wsv) noexcept;
 size_t UTF8PositionFromUTF16Position(std::string_view u8Text, size_t positionUTF16) noexcept;
 void UTF8FromUTF16(std::wstring_view wsv, char *putf, size_t len);
-void UTF8FromUTF32Character(int uch, char *putf);
+void UTF8FromUTF32Character(int uch, char *putf) noexcept;
-size_t UTF16Length(std::string_view sv);
+size_t UTF16Length(std::string_view svu8) noexcept;
-size_t UTF16FromUTF8(std::string_view sv, wchar_t *tbuf, size_t tlen);
+size_t UTF16FromUTF8(std::string_view svu8, wchar_t *tbuf, size_t tlen);
-size_t UTF32FromUTF8(std::string_view sv, unsigned int *tbuf, size_t tlen);
+size_t UTF32Length(std::string_view svu8) noexcept;
+size_t UTF32FromUTF8(std::string_view svu8, unsigned int *tbuf, size_t tlen);
+// WStringFromUTF8 does the right thing when wchar_t is 2 or 4 bytes so
+// works on both Windows and Unix.
+std::wstring WStringFromUTF8(std::string_view svu8);
 unsigned int UTF16FromUTF32Character(unsigned int val, wchar_t *tbuf) noexcept;
-bool UTF8IsValid(std::string_view sv) noexcept;
+bool UTF8IsValid(std::string_view svu8) noexcept;
 std::string FixInvalidUTF8(const std::string &text);
 
 extern const unsigned char UTF8BytesOfLead[256];

scintilla/src/UniqueString.cxx

@@ -0,0 +1,56 @@
+// Scintilla source code edit control
+/** @file UniqueString.cxx
+ ** Define an allocator for UniqueString.
+ **/
+// Copyright 2017 by Neil Hodgson <neilh@scintilla.org>
+// The License.txt file describes the conditions under which this software may be distributed.
+
+#include <string_view>
+#include <vector>
+#include <algorithm>
+#include <memory>
+
+#include "UniqueString.h"
+
+namespace Scintilla {
+
+/// Equivalent to strdup but produces a std::unique_ptr<const char[]> allocation to go
+/// into collections.
+UniqueString UniqueStringCopy(const char *text) {
+	if (!text) {
+		return UniqueString();
+	}
+	const std::string_view sv(text);
+	std::unique_ptr<char[]> upcNew = std::make_unique<char[]>(sv.length() + 1);
+	sv.copy(upcNew.get(), sv.length());
+	return UniqueString(upcNew.release());
+}
+
+// A set of strings that always returns the same pointer for each string.
+
+UniqueStringSet::UniqueStringSet() noexcept = default;
+
+UniqueStringSet::~UniqueStringSet() {
+	strings.clear();
+}
+
+void UniqueStringSet::Clear() noexcept {
+	strings.clear();
+}
+
+const char *UniqueStringSet::Save(const char *text) {
+	if (!text)
+		return nullptr;
+
+	const std::string_view sv(text);
+	for (const UniqueString &us : strings) {
+		if (sv == us.get()) {
+			return us.get();
+		}
+	}
+
+	strings.push_back(UniqueStringCopy(text));
+	return strings.back().get();
+}
+
+}

scintilla/src/UniqueString.h

@@ -2,6 +2,8 @@
 /** @file UniqueString.h
  ** Define UniqueString, a unique_ptr based string type for storage in containers
  ** and an allocator for UniqueString.
+ ** Define UniqueStringSet which holds a set of strings, used to avoid holding many copies
+ ** of font names.
  **/
 // Copyright 2017 by Neil Hodgson <neilh@scintilla.org>
 // The License.txt file describes the conditions under which this software may be distributed.
@@ -11,19 +13,33 @@
 
 namespace Scintilla {
 
+constexpr bool IsNullOrEmpty(const char *text) noexcept {
+	return text == nullptr || *text == '\0';
+}
+
 using UniqueString = std::unique_ptr<const char[]>;
 
 /// Equivalent to strdup but produces a std::unique_ptr<const char[]> allocation to go
 /// into collections.
-inline UniqueString UniqueStringCopy(const char *text) {
+UniqueString UniqueStringCopy(const char *text);
-	if (!text) {
+
-		return UniqueString();
+// A set of strings that always returns the same pointer for each string.
-	}
+
-	const size_t len = strlen(text);
+class UniqueStringSet {
-	char *sNew = new char[len + 1];
+private:
-	std::copy(text, text + len + 1, sNew);
+	std::vector<UniqueString> strings;
-	return UniqueString(sNew);
+public:
-}
+	UniqueStringSet() noexcept;
+	// UniqueStringSet objects can not be copied.
+	UniqueStringSet(const UniqueStringSet &) = delete;
+	UniqueStringSet &operator=(const UniqueStringSet &) = delete;
+	// UniqueStringSet objects can be moved.
+	UniqueStringSet(UniqueStringSet &&) = default;
+	UniqueStringSet &operator=(UniqueStringSet &&) = default;
+	~UniqueStringSet();
+	void Clear() noexcept;
+	const char *Save(const char *text);
+};
 
 }