diff options
66 files changed, 5685 insertions, 1245 deletions
@@ -28,7 +28,7 @@ Other contributors, listed alphabetically, are: * Pierre Bourdon -- bugfixes * Hiram Chirino -- Scaml and Jade lexers * Ian Cooper -- VGL lexer -* David Corbett -- Inform lexers +* David Corbett -- Inform and Jasmin lexers * Leaf Corcoran -- MoonScript lexer * Christopher Creutzig -- MuPAD lexer * Daniël W. Crompton - Pike lexer @@ -115,12 +115,14 @@ Other contributors, listed alphabetically, are: * Jesper Noehr -- HTML formatter "anchorlinenos" * Mike Nolta -- Julia lexer * Jonas Obrist -- BBCode lexer +* Edward O'Callaghan -- Cryptol lexer * David Oliva -- Rebol lexer * Pat Pannuto -- nesC lexer * Jon Parise -- Protocol buffers lexer * Ronny Pfannschmidt -- BBCode lexer * Benjamin Peterson -- Test suite refactoring * Dominik Picheta -- Nimrod lexer +* Andrew Pinkham -- RTF Formatter Refactoring * Clément Prévost -- UrbiScript lexer * raichoo -- Idris lexer * Kashif Rasul -- CUDA lexer @@ -157,6 +159,7 @@ Other contributors, listed alphabetically, are: * Pepijn de Vos -- HTML formatter CTags support * Whitney Young -- ObjectiveC lexer * Matthias Vallentin -- Bro lexer +* Linh Vu Hong -- RSL lexer * Nathan Weizenbaum -- Haml and Sass lexers * Dietmar Winkler -- Modelica lexer * Nils Winter -- Smalltalk lexer diff --git a/doc/languages.rst b/doc/languages.rst index 426a576b..0f98c583 100644 --- a/doc/languages.rst +++ b/doc/languages.rst @@ -25,6 +25,7 @@ Programming languages * ColdFusion * Common Lisp * Coq +* Cryptol (incl. Literate Cryptol) * `Cython <http://cython.org>`_ * `D <http://digitalmars.com/d>`_ * Dart @@ -65,7 +66,8 @@ Programming languages * PowerShell * Prolog * `Python <http://www.python.org>`_ 2.x and 3.x (incl. console sessions and tracebacks) -* Rebol +* `REBOL <http://www.rebol.com>`_ +* `Red <http://www.red-lang.org>`_ * Redcode * `Ruby <http://www.ruby-lang.org>`_ (incl. irb sessions) * Rust diff --git a/pygments/formatters/_mapping.py b/pygments/formatters/_mapping.py index 79f592b3..4c05ac8e 100755 --- a/pygments/formatters/_mapping.py +++ b/pygments/formatters/_mapping.py @@ -25,6 +25,7 @@ from pygments.formatters.img import JpgImageFormatter from pygments.formatters.latex import LatexFormatter from pygments.formatters.other import NullFormatter from pygments.formatters.other import RawTokenFormatter +from pygments.formatters.other import TestcaseFormatter from pygments.formatters.rtf import RtfFormatter from pygments.formatters.svg import SvgFormatter from pygments.formatters.terminal import TerminalFormatter @@ -43,7 +44,8 @@ FORMATTERS = { RtfFormatter: ('RTF', ('rtf',), ('*.rtf',), 'Format tokens as RTF markup. This formatter automatically outputs full RTF documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft\xc2\xae Word\xc2\xae documents.'), SvgFormatter: ('SVG', ('svg',), ('*.svg',), 'Format tokens as an SVG graphics file. This formatter is still experimental. Each line of code is a ``<text>`` element with explicit ``x`` and ``y`` coordinates containing ``<tspan>`` elements with the individual token styles.'), Terminal256Formatter: ('Terminal256', ('terminal256', 'console256', '256'), (), 'Format tokens with ANSI color sequences, for output in a 256-color terminal or console. Like in `TerminalFormatter` color sequences are terminated at newlines, so that paging the output works correctly.'), - TerminalFormatter: ('Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.') + TerminalFormatter: ('Terminal', ('terminal', 'console'), (), 'Format tokens with ANSI color sequences, for output in a text console. Color sequences are terminated at newlines, so that paging the output works correctly.'), + TestcaseFormatter: ('Testcase', ('testcase',), (), 'Format tokens as appropriate for a new testcase.') } if __name__ == '__main__': diff --git a/pygments/formatters/html.py b/pygments/formatters/html.py index 3bc60e8a..970e595b 100644 --- a/pygments/formatters/html.py +++ b/pygments/formatters/html.py @@ -523,7 +523,8 @@ class HtmlFormatter(Formatter): self.cssfile) except AttributeError: print('Note: Cannot determine output file name, ' \ - 'using current directory as base for the CSS file name', file=sys.stderr) + 'using current directory as base for the CSS file name', + file=sys.stderr) cssfilename = self.cssfile # write CSS file only if noclobber_cssfile isn't given as an option. try: diff --git a/pygments/formatters/latex.py b/pygments/formatters/latex.py index 413cca63..352684b0 100644 --- a/pygments/formatters/latex.py +++ b/pygments/formatters/latex.py @@ -375,9 +375,9 @@ class LatexFormatter(Formatter): if len(sep1) > 0: b,sep2,text = text.partition(self.right) if len(sep2) > 0: - value = value + escape_tex(a, self.commandprefix) + b + value += escape_tex(a, self.commandprefix) + b else: - value = value + escape_tex(a + sep1 + b, self.commandprefix) + value += escape_tex(a + sep1 + b, self.commandprefix) else: value = value + escape_tex(a, self.commandprefix) else: diff --git a/pygments/formatters/other.py b/pygments/formatters/other.py index 7368a642..c8269b19 100644 --- a/pygments/formatters/other.py +++ b/pygments/formatters/other.py @@ -14,7 +14,7 @@ from pygments.util import OptionError, get_choice_opt from pygments.token import Token from pygments.console import colorize -__all__ = ['NullFormatter', 'RawTokenFormatter'] +__all__ = ['NullFormatter', 'RawTokenFormatter', 'TestcaseFormatter'] class NullFormatter(Formatter): @@ -114,3 +114,49 @@ class RawTokenFormatter(Formatter): for ttype, value in tokensource: write("%s\t%r\n" % (ttype, value)) flush() + +TESTCASE_BEFORE = u'''\ + def testNeedsName(self): + fragment = %r + tokens = [ +''' +TESTCASE_AFTER = u'''\ + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(fragment))) +''' + + +class TestcaseFormatter(Formatter): + """ + Format tokens as appropriate for a new testcase. + + .. versionadded:: 2.0 + """ + name = 'Testcase' + aliases = ['testcase'] + + def __init__(self, **options): + Formatter.__init__(self, **options) + #if self.encoding != 'utf-8': + # print >>sys.stderr, "NOTICE: Forcing encoding to utf-8, as all Pygments source is" + if self.encoding is not None and self.encoding != 'utf-8': + raise ValueError("Only None and utf-u are allowed encodings.") + + def format(self, tokensource, outfile): + indentation = ' ' * 12 + rawbuf = [] + outbuf = [] + for ttype, value in tokensource: + rawbuf.append(value) + outbuf.append('%s(%s, %r),\n' % (indentation, ttype, value)) + + before = TESTCASE_BEFORE % (u''.join(rawbuf),) + during = u''.join(outbuf) + after = TESTCASE_AFTER + if self.encoding is None: + outfile.write(before + during + after) + else: + outfile.write(before.encode('utf-8')) + outfile.write(during.encode('utf-8')) + outfile.write(after.encode('utf-8')) + outfile.flush() diff --git a/pygments/formatters/rtf.py b/pygments/formatters/rtf.py index 9d87e8f1..cf65a927 100644 --- a/pygments/formatters/rtf.py +++ b/pygments/formatters/rtf.py @@ -10,7 +10,7 @@ """ from pygments.formatter import Formatter -from pygments.util import get_int_opt +from pygments.util import get_int_opt, _surrogatepair __all__ = ['RtfFormatter'] @@ -22,6 +22,10 @@ class RtfFormatter(Formatter): documents with color information and other useful stuff. Perfect for Copy and Paste into Microsoft® Word® documents. + Please note that ``encoding`` and ``outencoding`` options are ignored. + The RTF format is ASCII natively, but handles unicode characters correctly + thanks to escape sequences. + .. versionadded:: 0.6 Additional options accepted: @@ -74,28 +78,27 @@ class RtfFormatter(Formatter): # escape text text = self._escape(text) - if self.encoding in ('utf-8', 'utf-16', 'utf-32'): - encoding = 'iso-8859-15' - else: - encoding = self.encoding or 'iso-8859-15' buf = [] for c in text: - if ord(c) > 128: - ansic = c.encode(encoding, 'ignore') or '?' - if ord(ansic) > 128: - ansic = '\\\'%x' % ord(ansic) - else: - ansic = c - buf.append(r'\ud{\u%d%s}' % (ord(c), ansic)) - else: + cn = ord(c) + if cn < (2**7): + # ASCII character buf.append(str(c)) + elif (2**7) <= cn < (2**16): + # single unicode escape sequence + buf.append(r'{\u%d}' % cn) + elif (2**16) <= cn: + # RTF limits unicode to 16 bits. + # Force surrogate pairs + h,l = _surrogatepair(cn) + buf.append(r'{\u%d}{\u%d}' % (h,l)) return ''.join(buf).replace('\n', '\\par\n') def format_unencoded(self, tokensource, outfile): # rtf 1.8 header - outfile.write(r'{\rtf1\ansi\deff0' + outfile.write(r'{\rtf1\ansi\uc0\deff0' r'{\fonttbl{\f0\fmodern\fprq1\fcharset0%s;}}' r'{\colortbl;' % (self.fontface and ' ' + self._escape(self.fontface) or @@ -114,7 +117,7 @@ class RtfFormatter(Formatter): int(color[4:6], 16) )) offset += 1 - outfile.write(r'}\f0') + outfile.write(r'}\f0 ') if self.fontsize: outfile.write(r'\fs%d' % (self.fontsize)) diff --git a/pygments/lexers/_cocoabuiltins.py b/pygments/lexers/_cocoabuiltins.py index 1bfa0cdf..26179594 100644 --- a/pygments/lexers/_cocoabuiltins.py +++ b/pygments/lexers/_cocoabuiltins.py @@ -14,8 +14,8 @@ from __future__ import print_function -COCOA_INTERFACES = set(['UITableViewCell', 'NSURLSessionDataTask', 'NSLinguisticTagger', 'NSStream', 'UIPrintInfo', 'SKPaymentTransaction', 'SKPhysicsWorld', 'NSString', 'CMAttitude', 'SKSpriteNode', 'JSContext', 'UICollectionReusableView', 'AVMutableCompositionTrack', 'GKLeaderboard', 'NSFetchedResultsController', 'MKTileOverlayRenderer', 'MIDINetworkSession', 'UITextSelectionRect', 'MKRoute', 'MPVolumeView', 'UIKeyCommand', 'AVMutableAudioMix', 'GLKEffectPropertyLight', 'UICollectionViewLayout', 'NSMutableCharacterSet', 'UIAccessibilityElement', 'NSShadow', 'NSAtomicStoreCacheNode', 'UIPushBehavior', 'CBCharacteristic', 'CBUUID', 'CMStepCounter', 'NSNetService', 'UICollectionView', 'UIViewPrintFormatter', 'CAShapeLayer', 'MCPeerID', 'NSFileVersion', 'CMGyroData', 'SKPhysicsJointSpring', 'CIFilter', 'UIView', 'MKMapItem', 'PKPass', 'MKPolygonRenderer', 'JSValue', 'CLGeocoder', 'NSByteCountFormatter', 'AVCaptureScreenInput', 'CAAnimation', 'MKOverlayPathView', 'UIActionSheet', 'UIMotionEffectGroup', 'UIBarItem', 'SKProduct', 'AVAssetExportSession', 'NSKeyedUnarchiver', 'NSMutableSet', 'MKMapView', 'CATransition', 'CLCircularRegion', 'MKTileOverlay', 'UICollisionBehavior', 'ACAccountCredential', 'SKPhysicsJointLimit', 'AVMediaSelectionGroup', 'NSIndexSet', 'AVAudioRecorder', 'NSURL', 'CBCentral', 'NSNumber', 'UITableView', 'AVCaptureStillImageOutput', 'GCController', 'NSAssertionHandler', 'AVAudioSessionPortDescription', 'NSHTTPURLResponse', 'NSPropertyListSerialization', 'AVPlayerItemAccessLogEvent', 'UISwipeGestureRecognizer', 'MKOverlayRenderer', 'NSDecimalNumber', 'EKReminder', 'MKPolylineView', 'AVCaptureMovieFileOutput', 'UIImagePickerController', 'GKAchievementDescription', 'EKParticipant', 'NSBlockOperation', 'UIActivityItemProvider', 'CLLocation', 'GKLeaderboardViewController', 'MPMoviePlayerController', 'GKScore', 'NSURLConnection', 'ABUnknownPersonViewController', 'UIMenuController', 'NSEvent', 'SKTextureAtlas', 'NSKeyedArchiver', 'GKLeaderboardSet', 'NSSimpleCString', 'CBATTRequest', 'GKMatchRequest', 'AVMetadataObject', 'UIAlertView', 'NSIncrementalStore', 'MFMailComposeViewController', 'SSReadingList', 'MPMovieAccessLog', 'NSManagedObjectContext', 'AVCaptureAudioDataOutput', 'ACAccount', 'AVMetadataItem', 'AVCaptureDeviceInputSource', 'CLLocationManager', 'UIStepper', 'UIRefreshControl', 'GKTurnBasedParticipant', 'UICollectionViewTransitionLayout', 'CBCentralManager', 'NSPurgeableData', 'SLComposeViewController', 'NSHashTable', 'MKUserTrackingBarButtonItem', 'UITabBarController', 'CMMotionActivity', 'SKAction', 'AVPlayerItemOutput', 'UIDocumentInteractionController', 'UIDynamicItemBehavior', 'NSMutableDictionary', 'UILabel', 'AVCaptureInputPort', 'NSExpression', 'SKMutablePayment', 'UIStoryboardSegue', 'NSOrderedSet', 'UIPopoverBackgroundView', 'UIToolbar', 'NSNotificationCenter', 'NSEntityMigrationPolicy', 'NSLocale', 'NSURLSession', 'NSTimeZone', 'UIManagedDocument', 'AVMutableVideoCompositionLayerInstruction', 'AVAssetTrackGroup', 'NSInvocationOperation', 'ALAssetRepresentation', 'AVQueuePlayer', 'UIPasteboard', 'NSLayoutManager', 'EKCalendarChooser', 'EKObject', 'CATiledLayer', 'GLKReflectionMapEffect', 'NSManagedObjectID', 'NSUserDefaults', 'SLRequest', 'AVPlayerLayer', 'NSPointerArray', 'AVAudioMix', 'MCAdvertiserAssistant', 'MKMapSnapshotOptions', 'GKMatch', 'AVTimedMetadataGroup', 'CBMutableCharacteristic', 'NSFetchRequest', 'UIDevice', 'NSManagedObject', 'NKAssetDownload', 'AVOutputSettingsAssistant', 'SKPhysicsJointPin', 'UITabBar', 'UITextInputMode', 'NSFetchRequestExpression', 'NSPipe', 'AVComposition', 'ADBannerView', 'AVPlayerItem', 'AVSynchronizedLayer', 'MKDirectionsRequest', 'NSMetadataItem', 'UINavigationItem', 'CBPeripheralManager', 'UIStoryboardPopoverSegue', 'SKProductsRequest', 'UIGravityBehavior', 'UIWindow', 'CBMutableDescriptor', 'UIBezierPath', 'UINavigationController', 'ABPeoplePickerNavigationController', 'EKSource', 'AVAssetWriterInput', 'AVPlayerItemTrack', 'GLKEffectPropertyTexture', 'NSURLResponse', 'SKPaymentQueue', 'MKReverseGeocoder', 'GCControllerAxisInput', 'MKMapSnapshotter', 'NSOrthography', 'NSURLSessionUploadTask', 'NSCharacterSet', 'AVAssetReaderOutput', 'EAGLContext', 'UICollectionViewController', 'AVAssetTrack', 'SKEmitterNode', 'AVCaptureDeviceInput', 'AVVideoCompositionCoreAnimationTool', 'NSURLRequest', 'CMAccelerometerData', 'NSNetServiceBrowser', 'AVAsynchronousVideoCompositionRequest', 'CAGradientLayer', 'NSFormatter', 'CATransaction', 'MPMovieAccessLogEvent', 'UIStoryboard', 'MPMediaLibrary', 'UITapGestureRecognizer', 'MPMediaItemArtwork', 'NSURLSessionTask', 'MCBrowserViewController', 'NSRelationshipDescription', 'NSMutableAttributedString', 'MPNowPlayingInfoCenter', 'MKLocalSearch', 'EAAccessory', 'MKETAResponse', 'CATextLayer', 'NSNotificationQueue', 'NSValue', 'NSMutableIndexSet', 'SKPhysicsContact', 'NSProgress', 'CAScrollLayer', 'NSTextCheckingResult', 'NSEntityDescription', 'NSURLCredentialStorage', 'UIApplication', 'SKDownload', 'MKLocalSearchRequest', 'SKScene', 'UISearchDisplayController', 'CAReplicatorLayer', 'UIPrintPageRenderer', 'EKCalendarItem', 'NSUUID', 'EAAccessoryManager', 'AVAssetResourceLoader', 'AVMutableVideoCompositionInstruction', 'MyClass', 'CTCall', 'CIVector', 'UINavigationBar', 'UIPanGestureRecognizer', 'MPMediaQuery', 'ABNewPersonViewController', 'ACAccountType', 'GKSession', 'SKVideoNode', 'GCExtendedGamepadSnapshot', 'GCExtendedGamepad', 'CAValueFunction', 'UIActivityIndicatorView', 'NSNotification', 'SKReceiptRefreshRequest', 'AVCaptureDeviceFormat', 'AVPlayerItemErrorLog', 'NSMapTable', 'NSSet', 'CMMotionManager', 'GKVoiceChatService', 'UIPageControl', 'MKGeodesicPolyline', 'AVMutableComposition', 'NSLayoutConstraint', 'UIWebView', 'NSIncrementalStoreNode', 'EKEventStore', 'UISlider', 'AVAssetResourceLoadingRequest', 'AVCaptureInput', 'SKPhysicsBody', 'NSOperation', 'MKMapCamera', 'SKProductsResponse', 'GLKEffectPropertyMaterial', 'AVCaptureDevice', 'CTCallCenter', 'CBMutableService', 'SKTransition', 'UIDynamicAnimator', 'NSMutableArray', 'MCNearbyServiceBrowser', 'NSOperationQueue', 'MKPolylineRenderer', 'UICollectionViewLayoutAttributes', 'NSValueTransformer', 'UICollectionViewFlowLayout', 'NSEntityMapping', 'SKTexture', 'NSMergePolicy', 'UITextInputStringTokenizer', 'NSRecursiveLock', 'AVAsset', 'NSUndoManager', 'MPMediaPickerController', 'NSFileCoordinator', 'NSFileHandle', 'NSConditionLock', 'UISegmentedControl', 'NSManagedObjectModel', 'UITabBarItem', 'MPMediaItem', 'EKRecurrenceRule', 'UIEvent', 'UITouch', 'UIPrintInteractionController', 'CMDeviceMotion', 'NSCompoundPredicate', 'MKMultiPoint', 'UIPrintFormatter', 'SKView', 'NSConstantString', 'UIPopoverController', 'AVMetadataFaceObject', 'EKEventViewController', 'NSPort', 'MKCircleRenderer', 'AVCompositionTrack', 'UINib', 'NSUbiquitousKeyValueStore', 'NSMetadataQueryResultGroup', 'AVAssetResourceLoadingDataRequest', 'UITableViewHeaderFooterView', 'UISplitViewController', 'AVAudioSession', 'CAEmitterLayer', 'NSNull', 'MKCircleView', 'UIColor', 'UIAttachmentBehavior', 'CLBeacon', 'NSInputStream', 'NSURLCache', 'GKPlayer', 'NSMappingModel', 'NSHTTPCookie', 'AVMutableVideoComposition', 'NSAttributeDescription', 'AVPlayer', 'MKAnnotationView', 'UIFontDescriptor', 'NSTimer', 'CBDescriptor', 'MKOverlayView', 'EKEventEditViewController', 'NSSaveChangesRequest', 'UIReferenceLibraryViewController', 'SKPhysicsJointFixed', 'UILocalizedIndexedCollation', 'UIInterpolatingMotionEffect', 'AVAssetWriter', 'NSBundle', 'SKStoreProductViewController', 'GLKViewController', 'NSMetadataQueryAttributeValueTuple', 'GKTurnBasedMatch', 'UIActivity', 'MKShape', 'NSMergeConflict', 'CIImage', 'UIRotationGestureRecognizer', 'AVPlayerItemLegibleOutput', 'AVAssetImageGenerator', 'GCControllerButtonInput', 'NSSortDescriptor', 'MPTimedMetadata', 'NKIssue', 'UIScreenMode', 'GKTurnBasedEventHandler', 'MKPolyline', 'JSVirtualMachine', 'AVAssetReader', 'NSAttributedString', 'GKMatchmakerViewController', 'NSCountedSet', 'UIButton', 'GKLocalPlayer', 'MPMovieErrorLog', 'AVSpeechUtterance', 'AVURLAsset', 'CBPeripheral', 'AVAssetWriterInputGroup', 'AVAssetReaderAudioMixOutput', 'NSEnumerator', 'UIDocument', 'MKLocalSearchResponse', 'UISimpleTextPrintFormatter', 'CBService', 'MCSession', 'QLPreviewController', 'CAMediaTimingFunction', 'UITextPosition', 'NSNumberFormatter', 'UIPinchGestureRecognizer', 'UIMarkupTextPrintFormatter', 'MKRouteStep', 'NSMetadataQuery', 'AVAssetResourceLoadingContentInformationRequest', 'CTSubscriber', 'CTCarrier', 'NSFileSecurity', 'UIAcceleration', 'UIMotionEffect', 'CLHeading', 'NSFileWrapper', 'MKDirectionsResponse', 'UILocalNotification', 'UICollectionViewCell', 'UITextView', 'CMMagnetometerData', 'UIProgressView', 'GKInvite', 'UISearchBar', 'MKPlacemark', 'AVCaptureConnection', 'ALAssetsFilter', 'AVPlayerItemErrorLogEvent', 'NSJSONSerialization', 'AVAssetReaderVideoCompositionOutput', 'ABPersonViewController', 'CIDetector', 'GKTurnBasedMatchmakerViewController', 'MPMediaItemCollection', 'NSCondition', 'NSURLCredential', 'MIDINetworkConnection', 'NSDecimalNumberHandler', 'NSURLSessionConfiguration', 'EKCalendar', 'NSDictionary', 'CAPropertyAnimation', 'UIPercentDrivenInteractiveTransition', 'MKPolygon', 'AVAssetTrackSegment', 'NSExpressionDescription', 'UIViewController', 'NSURLAuthenticationChallenge', 'NSDirectoryEnumerator', 'MKDistanceFormatter', 'GCControllerElement', 'GKPeerPickerController', 'UITableViewController', 'GKNotificationBanner', 'MKPointAnnotation', 'NSCache', 'SKPhysicsJoint', 'NSXMLParser', 'MFMessageComposeViewController', 'AVCaptureSession', 'NSDataDetector', 'AVCaptureVideoPreviewLayer', 'NSURLComponents', 'UISnapBehavior', 'AVMetadataMachineReadableCodeObject', 'GLKTextureLoader', 'NSTextAttachment', 'NSException', 'UIMenuItem', 'CMMotionActivityManager', 'MKUserLocation', 'CIFeature', 'NSMachPort', 'ALAsset', 'NSURLSessionDownloadTask', 'MPMoviePlayerViewController', 'NSMutableOrderedSet', 'AVCaptureVideoDataOutput', 'NSCachedURLResponse', 'ALAssetsLibrary', 'NSInvocation', 'UILongPressGestureRecognizer', 'NSTextStorage', 'CIFaceFeature', 'MKMapSnapshot', 'GLKEffectPropertyFog', 'NSPersistentStoreRequest', 'AVAudioMixInputParameters', 'CAEmitterBehavior', 'PKPassLibrary', 'NSLock', 'UIDynamicBehavior', 'AVPlayerMediaSelectionCriteria', 'CALayer', 'UIBarButtonItem', 'AVAudioSessionRouteDescription', 'CLBeaconRegion', 'SKEffectNode', 'CABasicAnimation', 'AVVideoCompositionInstruction', 'AVMutableTimedMetadataGroup', 'EKRecurrenceEnd', 'NSTextContainer', 'TWTweetComposeViewController', 'UIScrollView', 'EKRecurrenceDayOfWeek', 'ASIdentifierManager', 'UIScreen', 'CLRegion', 'NSProcessInfo', 'GLKTextureInfo', 'AVCaptureMetadataOutput', 'NSTextTab', 'JSManagedValue', 'NSDate', 'UITextChecker', 'NSData', 'NSParagraphStyle', 'AVMutableMetadataItem', 'EKAlarm', 'NSMutableURLRequest', 'UIVideoEditorController', 'NSAtomicStore', 'UIResponder', 'AVCompositionTrackSegment', 'GCGamepadSnapshot', 'MPMediaEntity', 'GLKSkyboxEffect', 'UISwitch', 'EKStructuredLocation', 'UIGestureRecognizer', 'NSProxy', 'GLKBaseEffect', 'GKScoreChallenge', 'NSCoder', 'MPMediaPlaylist', 'NSDateComponents', 'EKEvent', 'NSDateFormatter', 'AVAssetWriterInputPixelBufferAdaptor', 'UICollectionViewFlowLayoutInvalidationContext', 'UITextField', 'CLPlacemark', 'AVCaptureOutput', 'NSPropertyDescription', 'GCGamepad', 'NSPersistentStoreCoordinator', 'GKMatchmaker', 'CIContext', 'NSThread', 'SKRequest', 'SKPhysicsJointSliding', 'NSPredicate', 'GKVoiceChat', 'SKCropNode', 'AVCaptureAudioPreviewOutput', 'NSStringDrawingContext', 'GKGameCenterViewController', 'UIPrintPaper', 'UICollectionViewLayoutInvalidationContext', 'GLKEffectPropertyTransform', 'UIDatePicker', 'MKDirections', 'ALAssetsGroup', 'CAEmitterCell', 'UIFont', 'MKPinAnnotationView', 'UIPickerView', 'UIImageView', 'SKNode', 'MPMediaQuerySection', 'GKFriendRequestComposeViewController', 'NSError', 'CTSubscriberInfo', 'AVPlayerItemAccessLog', 'MPMediaPropertyPredicate', 'CMLogItem', 'NSAutoreleasePool', 'NSSocketPort', 'AVAssetReaderTrackOutput', 'AVSpeechSynthesisVoice', 'UIImage', 'AVCaptureAudioChannel', 'GKTurnBasedExchangeReply', 'AVVideoCompositionLayerInstruction', 'AVSpeechSynthesizer', 'GKChallengeEventHandler', 'AVCaptureFileOutput', 'UIControl', 'SKPayment', 'ADInterstitialAd', 'AVAudioSessionDataSourceDescription', 'NSArray', 'GCControllerDirectionPad', 'NSFileManager', 'AVMutableAudioMixInputParameters', 'UIScreenEdgePanGestureRecognizer', 'CAKeyframeAnimation', 'EASession', 'UIInputView', 'NSHTTPCookieStorage', 'NSPointerFunctions', 'AVMediaSelectionOption', 'NSRunLoop', 'CAAnimationGroup', 'MKCircle', 'NSMigrationManager', 'UICollectionViewUpdateItem', 'NSMutableData', 'NSMutableParagraphStyle', 'GLKEffectProperty', 'SKShapeNode', 'MPMovieErrorLogEvent', 'MKPolygonView', 'UIAccelerometer', 'NSScanner', 'GKAchievementChallenge', 'AVAudioPlayer', 'AVVideoComposition', 'NKLibrary', 'NSPersistentStore', 'NSPropertyMapping', 'GKChallenge', 'NSURLProtectionSpace', 'ACAccountStore', 'UITextRange', 'NSComparisonPredicate', 'NSOutputStream', 'PKAddPassesViewController', 'CTTelephonyNetworkInfo', 'AVTextStyleRule', 'NSFetchedPropertyDescription', 'UIPageViewController', 'CATransformLayer', 'MCNearbyServiceAdvertiser', 'NSObject', 'MPMusicPlayerController', 'MKOverlayPathRenderer', 'GKAchievement', 'AVCaptureAudioFileOutput', 'TWRequest', 'SKLabelNode', 'MIDINetworkHost', 'MPMediaPredicate', 'AVFrameRateRange', 'NSIndexPath', 'AVVideoCompositionRenderContext', 'CADisplayLink', 'CAEAGLLayer', 'NSMutableString', 'NSMessagePort', 'AVAudioSessionChannelDescription', 'GLKView', 'UIActivityViewController', 'GKAchievementViewController', 'NSURLProtocol', 'NSCalendar', 'SKKeyframeSequence', 'AVMetadataItemFilter', 'NSMethodSignature', 'NSRegularExpression', 'EAGLSharegroup', 'AVPlayerItemVideoOutput', 'CIColor', 'UIDictationPhrase']) -COCOA_PROTOCOLS = set(['SKStoreProductViewControllerDelegate', 'AVVideoCompositionInstruction', 'AVAudioSessionDelegate', 'GKMatchDelegate', 'NSFileManagerDelegate', 'UILayoutSupport', 'NSCopying', 'UIPrintInteractionControllerDelegate', 'QLPreviewControllerDataSource', 'SKProductsRequestDelegate', 'NSTextStorageDelegate', 'MCBrowserViewControllerDelegate', 'UIViewControllerTransitionCoordinatorContext', 'NSTextAttachmentContainer', 'NSDecimalNumberBehaviors', 'NSMutableCopying', 'UIViewControllerTransitioningDelegate', 'UIAlertViewDelegate', 'AVAudioPlayerDelegate', 'MKReverseGeocoderDelegate', 'NSCoding', 'UITextInputTokenizer', 'GKFriendRequestComposeViewControllerDelegate', 'UIActivityItemSource', 'NSCacheDelegate', 'UITableViewDelegate', 'GKAchievementViewControllerDelegate', 'EKEventEditViewDelegate', 'NSURLConnectionDelegate', 'GKPeerPickerControllerDelegate', 'UIGuidedAccessRestrictionDelegate', 'AVSpeechSynthesizerDelegate', 'MFMailComposeViewControllerDelegate', 'AVPlayerItemLegibleOutputPushDelegate', 'ADInterstitialAdDelegate', 'AVAssetResourceLoaderDelegate', 'UITabBarControllerDelegate', 'SKPaymentTransactionObserver', 'AVCaptureAudioDataOutputSampleBufferDelegate', 'UIInputViewAudioFeedback', 'GKChallengeListener', 'UIPickerViewDelegate', 'UIWebViewDelegate', 'UIApplicationDelegate', 'GKInviteEventListener', 'MPMediaPlayback', 'MyClassJavaScriptMethods', 'AVAsynchronousKeyValueLoading', 'QLPreviewItem', 'NSPortDelegate', 'SKRequestDelegate', 'SKPhysicsContactDelegate', 'UIPageViewControllerDataSource', 'AVPlayerItemOutputPushDelegate', 'UICollectionViewDelegate', 'UIImagePickerControllerDelegate', 'UIToolbarDelegate', 'UIViewControllerTransitionCoordinator', 'NSURLConnectionDataDelegate', 'MKOverlay', 'CBCentralManagerDelegate', 'JSExport', 'NSTextLayoutOrientationProvider', 'UIPickerViewDataSource', 'UITextInputTraits', 'NSLayoutManagerDelegate', 'NSFetchedResultsControllerDelegate', 'ABPeoplePickerNavigationControllerDelegate', 'NSDiscardableContent', 'UITextFieldDelegate', 'GKGameCenterControllerDelegate', 'MPMediaPickerControllerDelegate', 'UIAppearance', 'UIPickerViewAccessibilityDelegate', 'UIScrollViewAccessibilityDelegate', 'ADBannerViewDelegate', 'NSURLSessionDelegate', 'NSXMLParserDelegate', 'UIViewControllerRestoration', 'UISearchBarDelegate', 'UIBarPositioning', 'CBPeripheralDelegate', 'UISearchDisplayDelegate', 'CAAction', 'PKAddPassesViewControllerDelegate', 'MCNearbyServiceAdvertiserDelegate', 'GKTurnBasedMatchmakerViewControllerDelegate', 'UIActionSheetDelegate', 'AVCaptureVideoDataOutputSampleBufferDelegate', 'UIAppearanceContainer', 'UIStateRestoring', 'NSURLSessionTaskDelegate', 'NSFilePresenter', 'UIViewControllerContextTransitioning', 'UITextInput', 'CBPeripheralManagerDelegate', 'UITextInputDelegate', 'NSFastEnumeration', 'NSURLAuthenticationChallengeSender', 'AVVideoCompositing', 'NSSecureCoding', 'MCAdvertiserAssistantDelegate', 'GKLocalPlayerListener', 'GLKNamedEffect', 'UIPopoverControllerDelegate', 'AVCaptureMetadataOutputObjectsDelegate', 'MFMessageComposeViewControllerDelegate', 'UITextSelecting', 'NSURLProtocolClient', 'UIVideoEditorControllerDelegate', 'UITableViewDataSource', 'UIDynamicAnimatorDelegate', 'NSURLSessionDataDelegate', 'UICollisionBehaviorDelegate', 'NSStreamDelegate', 'MCNearbyServiceBrowserDelegate', 'UINavigationControllerDelegate', 'MCSessionDelegate', 'UIViewControllerInteractiveTransitioning', 'GKTurnBasedEventListener', 'GLKViewDelegate', 'EAAccessoryDelegate', 'NSKeyedUnarchiverDelegate', 'NSMachPortDelegate', 'UIBarPositioningDelegate', 'ABPersonViewControllerDelegate', 'NSNetServiceBrowserDelegate', 'EKEventViewDelegate', 'UIScrollViewDelegate', 'NSURLConnectionDownloadDelegate', 'UIGestureRecognizerDelegate', 'UINavigationBarDelegate', 'GKVoiceChatClient', 'NSFetchedResultsSectionInfo', 'UIDocumentInteractionControllerDelegate', 'QLPreviewControllerDelegate', 'UIAccessibilityReadingContent', 'ABUnknownPersonViewControllerDelegate', 'GLKViewControllerDelegate', 'UICollectionViewDelegateFlowLayout', 'UISplitViewControllerDelegate', 'MKAnnotation', 'UIAccessibilityIdentification', 'ABNewPersonViewControllerDelegate', 'CAMediaTiming', 'AVCaptureFileOutputRecordingDelegate', 'UITextViewDelegate', 'UITabBarDelegate', 'GKLeaderboardViewControllerDelegate', 'MKMapViewDelegate', 'UIKeyInput', 'UICollectionViewDataSource', 'NSLocking', 'AVCaptureFileOutputDelegate', 'GKChallengeEventHandlerDelegate', 'UIObjectRestoration', 'CIFilterConstructor', 'AVPlayerItemOutputPullDelegate', 'EAGLDrawable', 'AVVideoCompositionValidationHandling', 'UIViewControllerAnimatedTransitioning', 'NSURLSessionDownloadDelegate', 'UIAccelerometerDelegate', 'UIPageViewControllerDelegate', 'UIDataSourceModelAssociation', 'AVAudioRecorderDelegate', 'GKSessionDelegate', 'NSKeyedArchiverDelegate', 'UIDynamicItem', 'CLLocationManagerDelegate', 'NSMetadataQueryDelegate', 'NSNetServiceDelegate', 'GKMatchmakerViewControllerDelegate', 'EKCalendarChooserDelegate']) +COCOA_INTERFACES = set(['UITableViewCell', 'NSURLSessionDataTask', 'NSLinguisticTagger', 'NSStream', 'UIPrintInfo', 'SKPaymentTransaction', 'SKPhysicsWorld', 'NSString', 'CMAttitude', 'SKSpriteNode', 'JSContext', 'UICollectionReusableView', 'AVMutableCompositionTrack', 'GKLeaderboard', 'NSFetchedResultsController', 'MKTileOverlayRenderer', 'MIDINetworkSession', 'UITextSelectionRect', 'MKRoute', 'MPVolumeView', 'UIKeyCommand', 'AVMutableAudioMix', 'GLKEffectPropertyLight', 'UICollectionViewLayout', 'NSMutableCharacterSet', 'UIAccessibilityElement', 'NSShadow', 'NSAtomicStoreCacheNode', 'UIPushBehavior', 'CBCharacteristic', 'CBUUID', 'CMStepCounter', 'NSNetService', 'UICollectionView', 'UIViewPrintFormatter', 'CAShapeLayer', 'MCPeerID', 'NSFileVersion', 'CMGyroData', 'SKPhysicsJointSpring', 'CIFilter', 'UIView', 'MKMapItem', 'PKPass', 'MKPolygonRenderer', 'JSValue', 'CLGeocoder', 'NSByteCountFormatter', 'AVCaptureScreenInput', 'CAAnimation', 'MKOverlayPathView', 'UIActionSheet', 'UIMotionEffectGroup', 'UIBarItem', 'SKProduct', 'AVAssetExportSession', 'NSKeyedUnarchiver', 'NSMutableSet', 'MKMapView', 'CATransition', 'CLCircularRegion', 'MKTileOverlay', 'UICollisionBehavior', 'ACAccountCredential', 'SKPhysicsJointLimit', 'AVMediaSelectionGroup', 'NSIndexSet', 'AVAudioRecorder', 'NSURL', 'CBCentral', 'NSNumber', 'UITableView', 'AVCaptureStillImageOutput', 'GCController', 'NSAssertionHandler', 'AVAudioSessionPortDescription', 'NSHTTPURLResponse', 'NSPropertyListSerialization', 'AVPlayerItemAccessLogEvent', 'UISwipeGestureRecognizer', 'MKOverlayRenderer', 'NSDecimalNumber', 'EKReminder', 'MKPolylineView', 'AVCaptureMovieFileOutput', 'UIImagePickerController', 'GKAchievementDescription', 'EKParticipant', 'NSBlockOperation', 'UIActivityItemProvider', 'CLLocation', 'GKLeaderboardViewController', 'MPMoviePlayerController', 'GKScore', 'NSURLConnection', 'ABUnknownPersonViewController', 'UIMenuController', 'NSEvent', 'SKTextureAtlas', 'NSKeyedArchiver', 'GKLeaderboardSet', 'NSSimpleCString', 'CBATTRequest', 'GKMatchRequest', 'AVMetadataObject', 'UIAlertView', 'NSIncrementalStore', 'MFMailComposeViewController', 'SSReadingList', 'MPMovieAccessLog', 'NSManagedObjectContext', 'AVCaptureAudioDataOutput', 'ACAccount', 'AVMetadataItem', 'AVCaptureDeviceInputSource', 'CLLocationManager', 'UIStepper', 'UIRefreshControl', 'GKTurnBasedParticipant', 'UICollectionViewTransitionLayout', 'CBCentralManager', 'NSPurgeableData', 'SLComposeViewController', 'NSHashTable', 'MKUserTrackingBarButtonItem', 'UITabBarController', 'CMMotionActivity', 'SKAction', 'AVPlayerItemOutput', 'UIDocumentInteractionController', 'UIDynamicItemBehavior', 'NSMutableDictionary', 'UILabel', 'AVCaptureInputPort', 'NSExpression', 'SKMutablePayment', 'UIStoryboardSegue', 'NSOrderedSet', 'UIPopoverBackgroundView', 'UIToolbar', 'NSNotificationCenter', 'NSEntityMigrationPolicy', 'NSLocale', 'NSURLSession', 'NSTimeZone', 'UIManagedDocument', 'AVMutableVideoCompositionLayerInstruction', 'AVAssetTrackGroup', 'NSInvocationOperation', 'ALAssetRepresentation', 'AVQueuePlayer', 'UIPasteboard', 'NSLayoutManager', 'EKCalendarChooser', 'EKObject', 'CATiledLayer', 'GLKReflectionMapEffect', 'NSManagedObjectID', 'NSUserDefaults', 'SLRequest', 'AVPlayerLayer', 'NSPointerArray', 'AVAudioMix', 'MCAdvertiserAssistant', 'MKMapSnapshotOptions', 'GKMatch', 'AVTimedMetadataGroup', 'CBMutableCharacteristic', 'NSFetchRequest', 'UIDevice', 'NSManagedObject', 'NKAssetDownload', 'AVOutputSettingsAssistant', 'SKPhysicsJointPin', 'UITabBar', 'UITextInputMode', 'NSFetchRequestExpression', 'NSPipe', 'AVComposition', 'ADBannerView', 'AVPlayerItem', 'AVSynchronizedLayer', 'MKDirectionsRequest', 'NSMetadataItem', 'UINavigationItem', 'CBPeripheralManager', 'UIStoryboardPopoverSegue', 'SKProductsRequest', 'UIGravityBehavior', 'UIWindow', 'CBMutableDescriptor', 'UIBezierPath', 'UINavigationController', 'ABPeoplePickerNavigationController', 'EKSource', 'AVAssetWriterInput', 'AVPlayerItemTrack', 'GLKEffectPropertyTexture', 'NSURLResponse', 'SKPaymentQueue', 'MKReverseGeocoder', 'GCControllerAxisInput', 'MKMapSnapshotter', 'NSOrthography', 'NSURLSessionUploadTask', 'NSCharacterSet', 'AVAssetReaderOutput', 'EAGLContext', 'UICollectionViewController', 'AVAssetTrack', 'SKEmitterNode', 'AVCaptureDeviceInput', 'AVVideoCompositionCoreAnimationTool', 'NSURLRequest', 'CMAccelerometerData', 'NSNetServiceBrowser', 'AVAsynchronousVideoCompositionRequest', 'CAGradientLayer', 'NSFormatter', 'CATransaction', 'MPMovieAccessLogEvent', 'UIStoryboard', 'MPMediaLibrary', 'UITapGestureRecognizer', 'MPMediaItemArtwork', 'NSURLSessionTask', 'MCBrowserViewController', 'NSRelationshipDescription', 'NSMutableAttributedString', 'MPNowPlayingInfoCenter', 'MKLocalSearch', 'EAAccessory', 'MKETAResponse', 'CATextLayer', 'NSNotificationQueue', 'NSValue', 'NSMutableIndexSet', 'SKPhysicsContact', 'NSProgress', 'CAScrollLayer', 'NSTextCheckingResult', 'NSEntityDescription', 'NSURLCredentialStorage', 'UIApplication', 'SKDownload', 'MKLocalSearchRequest', 'SKScene', 'UISearchDisplayController', 'CAReplicatorLayer', 'UIPrintPageRenderer', 'EKCalendarItem', 'NSUUID', 'EAAccessoryManager', 'AVAssetResourceLoader', 'AVMutableVideoCompositionInstruction', 'CTCall', 'CIVector', 'UINavigationBar', 'UIPanGestureRecognizer', 'MPMediaQuery', 'ABNewPersonViewController', 'ACAccountType', 'GKSession', 'SKVideoNode', 'GCExtendedGamepadSnapshot', 'GCExtendedGamepad', 'CAValueFunction', 'UIActivityIndicatorView', 'NSNotification', 'SKReceiptRefreshRequest', 'AVCaptureDeviceFormat', 'AVPlayerItemErrorLog', 'NSMapTable', 'NSSet', 'CMMotionManager', 'GKVoiceChatService', 'UIPageControl', 'MKGeodesicPolyline', 'AVMutableComposition', 'NSLayoutConstraint', 'UIWebView', 'NSIncrementalStoreNode', 'EKEventStore', 'UISlider', 'AVAssetResourceLoadingRequest', 'AVCaptureInput', 'SKPhysicsBody', 'NSOperation', 'MKMapCamera', 'SKProductsResponse', 'GLKEffectPropertyMaterial', 'AVCaptureDevice', 'CTCallCenter', 'CBMutableService', 'SKTransition', 'UIDynamicAnimator', 'NSMutableArray', 'MCNearbyServiceBrowser', 'NSOperationQueue', 'MKPolylineRenderer', 'UICollectionViewLayoutAttributes', 'NSValueTransformer', 'UICollectionViewFlowLayout', 'NSEntityMapping', 'SKTexture', 'NSMergePolicy', 'UITextInputStringTokenizer', 'NSRecursiveLock', 'AVAsset', 'NSUndoManager', 'MPMediaPickerController', 'NSFileCoordinator', 'NSFileHandle', 'NSConditionLock', 'UISegmentedControl', 'NSManagedObjectModel', 'UITabBarItem', 'MPMediaItem', 'EKRecurrenceRule', 'UIEvent', 'UITouch', 'UIPrintInteractionController', 'CMDeviceMotion', 'NSCompoundPredicate', 'MKMultiPoint', 'UIPrintFormatter', 'SKView', 'NSConstantString', 'UIPopoverController', 'AVMetadataFaceObject', 'EKEventViewController', 'NSPort', 'MKCircleRenderer', 'AVCompositionTrack', 'UINib', 'NSUbiquitousKeyValueStore', 'NSMetadataQueryResultGroup', 'AVAssetResourceLoadingDataRequest', 'UITableViewHeaderFooterView', 'UISplitViewController', 'AVAudioSession', 'CAEmitterLayer', 'NSNull', 'MKCircleView', 'UIColor', 'UIAttachmentBehavior', 'CLBeacon', 'NSInputStream', 'NSURLCache', 'GKPlayer', 'NSMappingModel', 'NSHTTPCookie', 'AVMutableVideoComposition', 'NSAttributeDescription', 'AVPlayer', 'MKAnnotationView', 'UIFontDescriptor', 'NSTimer', 'CBDescriptor', 'MKOverlayView', 'EKEventEditViewController', 'NSSaveChangesRequest', 'UIReferenceLibraryViewController', 'SKPhysicsJointFixed', 'UILocalizedIndexedCollation', 'UIInterpolatingMotionEffect', 'AVAssetWriter', 'NSBundle', 'SKStoreProductViewController', 'GLKViewController', 'NSMetadataQueryAttributeValueTuple', 'GKTurnBasedMatch', 'UIActivity', 'MKShape', 'NSMergeConflict', 'CIImage', 'UIRotationGestureRecognizer', 'AVPlayerItemLegibleOutput', 'AVAssetImageGenerator', 'GCControllerButtonInput', 'NSSortDescriptor', 'MPTimedMetadata', 'NKIssue', 'UIScreenMode', 'GKTurnBasedEventHandler', 'MKPolyline', 'JSVirtualMachine', 'AVAssetReader', 'NSAttributedString', 'GKMatchmakerViewController', 'NSCountedSet', 'UIButton', 'GKLocalPlayer', 'MPMovieErrorLog', 'AVSpeechUtterance', 'AVURLAsset', 'CBPeripheral', 'AVAssetWriterInputGroup', 'AVAssetReaderAudioMixOutput', 'NSEnumerator', 'UIDocument', 'MKLocalSearchResponse', 'UISimpleTextPrintFormatter', 'CBService', 'MCSession', 'QLPreviewController', 'CAMediaTimingFunction', 'UITextPosition', 'NSNumberFormatter', 'UIPinchGestureRecognizer', 'UIMarkupTextPrintFormatter', 'MKRouteStep', 'NSMetadataQuery', 'AVAssetResourceLoadingContentInformationRequest', 'CTSubscriber', 'CTCarrier', 'NSFileSecurity', 'UIAcceleration', 'UIMotionEffect', 'CLHeading', 'NSFileWrapper', 'MKDirectionsResponse', 'UILocalNotification', 'UICollectionViewCell', 'UITextView', 'CMMagnetometerData', 'UIProgressView', 'GKInvite', 'UISearchBar', 'MKPlacemark', 'AVCaptureConnection', 'ALAssetsFilter', 'AVPlayerItemErrorLogEvent', 'NSJSONSerialization', 'AVAssetReaderVideoCompositionOutput', 'ABPersonViewController', 'CIDetector', 'GKTurnBasedMatchmakerViewController', 'MPMediaItemCollection', 'NSCondition', 'NSURLCredential', 'MIDINetworkConnection', 'NSDecimalNumberHandler', 'NSURLSessionConfiguration', 'EKCalendar', 'NSDictionary', 'CAPropertyAnimation', 'UIPercentDrivenInteractiveTransition', 'MKPolygon', 'AVAssetTrackSegment', 'NSExpressionDescription', 'UIViewController', 'NSURLAuthenticationChallenge', 'NSDirectoryEnumerator', 'MKDistanceFormatter', 'GCControllerElement', 'GKPeerPickerController', 'UITableViewController', 'GKNotificationBanner', 'MKPointAnnotation', 'NSCache', 'SKPhysicsJoint', 'NSXMLParser', 'MFMessageComposeViewController', 'AVCaptureSession', 'NSDataDetector', 'AVCaptureVideoPreviewLayer', 'NSURLComponents', 'UISnapBehavior', 'AVMetadataMachineReadableCodeObject', 'GLKTextureLoader', 'NSTextAttachment', 'NSException', 'UIMenuItem', 'CMMotionActivityManager', 'MKUserLocation', 'CIFeature', 'NSMachPort', 'ALAsset', 'NSURLSessionDownloadTask', 'MPMoviePlayerViewController', 'NSMutableOrderedSet', 'AVCaptureVideoDataOutput', 'NSCachedURLResponse', 'ALAssetsLibrary', 'NSInvocation', 'UILongPressGestureRecognizer', 'NSTextStorage', 'CIFaceFeature', 'MKMapSnapshot', 'GLKEffectPropertyFog', 'NSPersistentStoreRequest', 'AVAudioMixInputParameters', 'CAEmitterBehavior', 'PKPassLibrary', 'NSLock', 'UIDynamicBehavior', 'AVPlayerMediaSelectionCriteria', 'CALayer', 'UIBarButtonItem', 'AVAudioSessionRouteDescription', 'CLBeaconRegion', 'SKEffectNode', 'CABasicAnimation', 'AVVideoCompositionInstruction', 'AVMutableTimedMetadataGroup', 'EKRecurrenceEnd', 'NSTextContainer', 'TWTweetComposeViewController', 'UIScrollView', 'EKRecurrenceDayOfWeek', 'ASIdentifierManager', 'UIScreen', 'CLRegion', 'NSProcessInfo', 'GLKTextureInfo', 'AVCaptureMetadataOutput', 'NSTextTab', 'JSManagedValue', 'NSDate', 'UITextChecker', 'NSData', 'NSParagraphStyle', 'AVMutableMetadataItem', 'EKAlarm', 'NSMutableURLRequest', 'UIVideoEditorController', 'NSAtomicStore', 'UIResponder', 'AVCompositionTrackSegment', 'GCGamepadSnapshot', 'MPMediaEntity', 'GLKSkyboxEffect', 'UISwitch', 'EKStructuredLocation', 'UIGestureRecognizer', 'NSProxy', 'GLKBaseEffect', 'GKScoreChallenge', 'NSCoder', 'MPMediaPlaylist', 'NSDateComponents', 'EKEvent', 'NSDateFormatter', 'AVAssetWriterInputPixelBufferAdaptor', 'UICollectionViewFlowLayoutInvalidationContext', 'UITextField', 'CLPlacemark', 'AVCaptureOutput', 'NSPropertyDescription', 'GCGamepad', 'NSPersistentStoreCoordinator', 'GKMatchmaker', 'CIContext', 'NSThread', 'SKRequest', 'SKPhysicsJointSliding', 'NSPredicate', 'GKVoiceChat', 'SKCropNode', 'AVCaptureAudioPreviewOutput', 'NSStringDrawingContext', 'GKGameCenterViewController', 'UIPrintPaper', 'UICollectionViewLayoutInvalidationContext', 'GLKEffectPropertyTransform', 'UIDatePicker', 'MKDirections', 'ALAssetsGroup', 'CAEmitterCell', 'UIFont', 'MKPinAnnotationView', 'UIPickerView', 'UIImageView', 'SKNode', 'MPMediaQuerySection', 'GKFriendRequestComposeViewController', 'NSError', 'CTSubscriberInfo', 'AVPlayerItemAccessLog', 'MPMediaPropertyPredicate', 'CMLogItem', 'NSAutoreleasePool', 'NSSocketPort', 'AVAssetReaderTrackOutput', 'AVSpeechSynthesisVoice', 'UIImage', 'AVCaptureAudioChannel', 'GKTurnBasedExchangeReply', 'AVVideoCompositionLayerInstruction', 'AVSpeechSynthesizer', 'GKChallengeEventHandler', 'AVCaptureFileOutput', 'UIControl', 'SKPayment', 'ADInterstitialAd', 'AVAudioSessionDataSourceDescription', 'NSArray', 'GCControllerDirectionPad', 'NSFileManager', 'AVMutableAudioMixInputParameters', 'UIScreenEdgePanGestureRecognizer', 'CAKeyframeAnimation', 'EASession', 'UIInputView', 'NSHTTPCookieStorage', 'NSPointerFunctions', 'AVMediaSelectionOption', 'NSRunLoop', 'CAAnimationGroup', 'MKCircle', 'NSMigrationManager', 'UICollectionViewUpdateItem', 'NSMutableData', 'NSMutableParagraphStyle', 'GLKEffectProperty', 'SKShapeNode', 'MPMovieErrorLogEvent', 'MKPolygonView', 'UIAccelerometer', 'NSScanner', 'GKAchievementChallenge', 'AVAudioPlayer', 'AVVideoComposition', 'NKLibrary', 'NSPersistentStore', 'NSPropertyMapping', 'GKChallenge', 'NSURLProtectionSpace', 'ACAccountStore', 'UITextRange', 'NSComparisonPredicate', 'NSOutputStream', 'PKAddPassesViewController', 'CTTelephonyNetworkInfo', 'AVTextStyleRule', 'NSFetchedPropertyDescription', 'UIPageViewController', 'CATransformLayer', 'MCNearbyServiceAdvertiser', 'NSObject', 'MPMusicPlayerController', 'MKOverlayPathRenderer', 'GKAchievement', 'AVCaptureAudioFileOutput', 'TWRequest', 'SKLabelNode', 'MIDINetworkHost', 'MPMediaPredicate', 'AVFrameRateRange', 'NSIndexPath', 'AVVideoCompositionRenderContext', 'CADisplayLink', 'CAEAGLLayer', 'NSMutableString', 'NSMessagePort', 'AVAudioSessionChannelDescription', 'GLKView', 'UIActivityViewController', 'GKAchievementViewController', 'NSURLProtocol', 'NSCalendar', 'SKKeyframeSequence', 'AVMetadataItemFilter', 'NSMethodSignature', 'NSRegularExpression', 'EAGLSharegroup', 'AVPlayerItemVideoOutput', 'CIColor', 'UIDictationPhrase']) +COCOA_PROTOCOLS = set(['SKStoreProductViewControllerDelegate', 'AVVideoCompositionInstruction', 'AVAudioSessionDelegate', 'GKMatchDelegate', 'NSFileManagerDelegate', 'UILayoutSupport', 'NSCopying', 'UIPrintInteractionControllerDelegate', 'QLPreviewControllerDataSource', 'SKProductsRequestDelegate', 'NSTextStorageDelegate', 'MCBrowserViewControllerDelegate', 'UIViewControllerTransitionCoordinatorContext', 'NSTextAttachmentContainer', 'NSDecimalNumberBehaviors', 'NSMutableCopying', 'UIViewControllerTransitioningDelegate', 'UIAlertViewDelegate', 'AVAudioPlayerDelegate', 'MKReverseGeocoderDelegate', 'NSCoding', 'UITextInputTokenizer', 'GKFriendRequestComposeViewControllerDelegate', 'UIActivityItemSource', 'NSCacheDelegate', 'UITableViewDelegate', 'GKAchievementViewControllerDelegate', 'EKEventEditViewDelegate', 'NSURLConnectionDelegate', 'GKPeerPickerControllerDelegate', 'UIGuidedAccessRestrictionDelegate', 'AVSpeechSynthesizerDelegate', 'MFMailComposeViewControllerDelegate', 'AVPlayerItemLegibleOutputPushDelegate', 'ADInterstitialAdDelegate', 'AVAssetResourceLoaderDelegate', 'UITabBarControllerDelegate', 'SKPaymentTransactionObserver', 'AVCaptureAudioDataOutputSampleBufferDelegate', 'UIInputViewAudioFeedback', 'GKChallengeListener', 'UIPickerViewDelegate', 'UIWebViewDelegate', 'UIApplicationDelegate', 'GKInviteEventListener', 'MPMediaPlayback', 'AVAsynchronousKeyValueLoading', 'QLPreviewItem', 'NSPortDelegate', 'SKRequestDelegate', 'SKPhysicsContactDelegate', 'UIPageViewControllerDataSource', 'AVPlayerItemOutputPushDelegate', 'UICollectionViewDelegate', 'UIImagePickerControllerDelegate', 'UIToolbarDelegate', 'UIViewControllerTransitionCoordinator', 'NSURLConnectionDataDelegate', 'MKOverlay', 'CBCentralManagerDelegate', 'JSExport', 'NSTextLayoutOrientationProvider', 'UIPickerViewDataSource', 'UITextInputTraits', 'NSLayoutManagerDelegate', 'NSFetchedResultsControllerDelegate', 'ABPeoplePickerNavigationControllerDelegate', 'NSDiscardableContent', 'UITextFieldDelegate', 'GKGameCenterControllerDelegate', 'MPMediaPickerControllerDelegate', 'UIAppearance', 'UIPickerViewAccessibilityDelegate', 'UIScrollViewAccessibilityDelegate', 'ADBannerViewDelegate', 'NSURLSessionDelegate', 'NSXMLParserDelegate', 'UIViewControllerRestoration', 'UISearchBarDelegate', 'UIBarPositioning', 'CBPeripheralDelegate', 'UISearchDisplayDelegate', 'CAAction', 'PKAddPassesViewControllerDelegate', 'MCNearbyServiceAdvertiserDelegate', 'GKTurnBasedMatchmakerViewControllerDelegate', 'UIActionSheetDelegate', 'AVCaptureVideoDataOutputSampleBufferDelegate', 'UIAppearanceContainer', 'UIStateRestoring', 'NSURLSessionTaskDelegate', 'NSFilePresenter', 'UIViewControllerContextTransitioning', 'UITextInput', 'CBPeripheralManagerDelegate', 'UITextInputDelegate', 'NSFastEnumeration', 'NSURLAuthenticationChallengeSender', 'AVVideoCompositing', 'NSSecureCoding', 'MCAdvertiserAssistantDelegate', 'GKLocalPlayerListener', 'GLKNamedEffect', 'UIPopoverControllerDelegate', 'AVCaptureMetadataOutputObjectsDelegate', 'MFMessageComposeViewControllerDelegate', 'UITextSelecting', 'NSURLProtocolClient', 'UIVideoEditorControllerDelegate', 'UITableViewDataSource', 'UIDynamicAnimatorDelegate', 'NSURLSessionDataDelegate', 'UICollisionBehaviorDelegate', 'NSStreamDelegate', 'MCNearbyServiceBrowserDelegate', 'UINavigationControllerDelegate', 'MCSessionDelegate', 'UIViewControllerInteractiveTransitioning', 'GKTurnBasedEventListener', 'GLKViewDelegate', 'EAAccessoryDelegate', 'NSKeyedUnarchiverDelegate', 'NSMachPortDelegate', 'UIBarPositioningDelegate', 'ABPersonViewControllerDelegate', 'NSNetServiceBrowserDelegate', 'EKEventViewDelegate', 'UIScrollViewDelegate', 'NSURLConnectionDownloadDelegate', 'UIGestureRecognizerDelegate', 'UINavigationBarDelegate', 'GKVoiceChatClient', 'NSFetchedResultsSectionInfo', 'UIDocumentInteractionControllerDelegate', 'QLPreviewControllerDelegate', 'UIAccessibilityReadingContent', 'ABUnknownPersonViewControllerDelegate', 'GLKViewControllerDelegate', 'UICollectionViewDelegateFlowLayout', 'UISplitViewControllerDelegate', 'MKAnnotation', 'UIAccessibilityIdentification', 'ABNewPersonViewControllerDelegate', 'CAMediaTiming', 'AVCaptureFileOutputRecordingDelegate', 'UITextViewDelegate', 'UITabBarDelegate', 'GKLeaderboardViewControllerDelegate', 'MKMapViewDelegate', 'UIKeyInput', 'UICollectionViewDataSource', 'NSLocking', 'AVCaptureFileOutputDelegate', 'GKChallengeEventHandlerDelegate', 'UIObjectRestoration', 'CIFilterConstructor', 'AVPlayerItemOutputPullDelegate', 'EAGLDrawable', 'AVVideoCompositionValidationHandling', 'UIViewControllerAnimatedTransitioning', 'NSURLSessionDownloadDelegate', 'UIAccelerometerDelegate', 'UIPageViewControllerDelegate', 'UIDataSourceModelAssociation', 'AVAudioRecorderDelegate', 'GKSessionDelegate', 'NSKeyedArchiverDelegate', 'UIDynamicItem', 'CLLocationManagerDelegate', 'NSMetadataQueryDelegate', 'NSNetServiceDelegate', 'GKMatchmakerViewControllerDelegate', 'EKCalendarChooserDelegate', 'NSTextField', 'NSInteger', 'NSUInteger']) COCOA_PRIMITIVES = set(['ROTAHeader', '__CFBundle', 'MortSubtable', 'AudioFilePacketTableInfo', 'CGPDFOperatorTable', 'KerxStateEntry', 'ExtendedTempoEvent', 'CTParagraphStyleSetting', 'OpaqueMIDIPort', 'CFStreamErrorHTTP', '__CFMachPort', '_GLKMatrix4', 'ExtendedControlEvent', 'CAFAudioDescription', 'KernVersion0Header', 'CGTextDrawingMode', 'EKErrorCode', 'gss_buffer_desc_struct', 'AudioUnitParameterInfo', '__SCPreferences', '__CTFrame', '__CTLine', 'CFStreamSocketSecurityProtocol', 'gss_krb5_lucid_context_v1', 'OpaqueJSValue', 'TrakTableEntry', 'AudioFramePacketTranslation', 'CGImageSource', 'OpaqueJSPropertyNameAccumulator', 'JustPCGlyphRepeatAddAction', 'BslnFormat0Part', 'OpaqueMIDIThruConnection', 'opaqueCMBufferQueue', 'OpaqueMusicSequence', 'MortRearrangementSubtable', 'MixerDistanceParams', 'MorxSubtable', 'MIDIObjectPropertyChangeNotification', '__CFDictionary', 'CGImageMetadataErrors', 'CGPath', 'OpaqueMIDIEndpoint', 'ALMXHeader', 'AudioComponentPlugInInterface', 'gss_ctx_id_t_desc_struct', 'sfntFontFeatureSetting', 'OpaqueJSContextGroup', '__SCNetworkConnection', 'AudioUnitParameterValueTranslation', 'CGImageMetadataType', 'CGPattern', 'AudioFileTypeAndFormatID', 'CGContext', 'AUNodeInteraction', 'SFNTLookupTable', 'JustPCDecompositionAction', 'KerxControlPointHeader', 'PKErrorCode', 'AudioStreamPacketDescription', 'KernSubtableHeader', '__CFNull', 'AUMIDIOutputCallbackStruct', 'MIDIMetaEvent', 'AudioQueueChannelAssignment', '__CFString', 'AnchorPoint', 'JustTable', '__CFNetService', 'gss_krb5_lucid_key', 'CGPDFDictionary', 'MIDIThruConnectionParams', 'CAF_UUID_ChunkHeader', 'gss_krb5_cfx_keydata', '_GLKMatrix3', 'CGGradient', 'OpaqueMIDISetup', '_GLKMatrix2', 'JustPostcompTable', '__CTParagraphStyle', 'AudioUnitParameterHistoryInfo', 'OpaqueJSContext', 'CGShading', '__CFBinaryHeap', 'SFNTLookupSingle', '__CFHost', '__SecRandom', '__CTFontDescriptor', '_NSRange', 'sfntDirectory', 'AudioQueueLevelMeterState', 'CAFPositionPeak', '__CFBoolean', 'PropLookupSegment', '__CVOpenGLESTextureCache', 'sfntInstance', '_GLKQuaternion', 'KernStateEntry', '__SCNetworkProtocol', 'CAFFileHeader', 'KerxOrderedListHeader', 'CGBlendMode', 'STXEntryOne', 'CAFRegion', 'SFNTLookupTrimmedArrayHeader', 'KerxControlPointEntry', '__CFCharacterSet', 'OpaqueMusicTrack', '_GLKVector4', 'gss_OID_set_desc_struct', 'OpaqueMusicPlayer', '_CFHTTPAuthentication', 'CGAffineTransform', 'CAFMarkerChunk', 'AUHostIdentifier', 'ROTAGlyphEntry', 'BslnTable', 'gss_krb5_lucid_context_version', '_GLKMatrixStack', 'CGImage', 'AnkrTable', 'SFNTLookupSingleHeader', 'MortLigatureSubtable', 'AudioFile_SMPTE_Time', 'CAFUMIDChunk', 'SMPTETime', 'CAFDataChunk', 'CGPDFStream', 'AudioFileRegionList', 'STEntryTwo', 'SFNTLookupBinarySearchHeader', 'OpbdTable', '__CTGlyphInfo', 'BslnFormat2Part', 'KerxIndexArrayHeader', 'TrakTable', 'KerxKerningPair', '__CFBitVector', 'KernVersion0SubtableHeader', 'OpaqueAudioComponentInstance', 'AudioChannelLayout', '__CFUUID', 'MIDISysexSendRequest', '__CFNumberFormatter', 'CGImageSourceStatus', '__CFURL', 'AudioFileMarkerList', 'AUSamplerBankPresetData', 'CGDataProvider', 'AudioFormatInfo', '__SecIdentity', 'sfntCMapExtendedSubHeader', 'MIDIChannelMessage', 'KernOffsetTable', 'CGColorSpaceModel', 'MFMailComposeErrorCode', 'CGFunction', '__SecTrust', 'CFHostInfoType', 'KernSimpleArrayHeader', 'CGFontPostScriptFormat', 'KernStateHeader', 'AudioUnitCocoaViewInfo', 'CGDataConsumer', 'OpaqueMIDIDevice', 'OpaqueCMBlockBuffer', 'AnchorPointTable', 'CGImageDestination', 'CAFInstrumentChunk', 'AudioUnitMeterClipping', '__CFNumber', 'MorxChain', '__CTFontCollection', 'STEntryOne', 'STXEntryTwo', 'ExtendedNoteOnEvent', '__CFArray', 'CGColorRenderingIntent', 'KerxSimpleArrayHeader', 'MorxTable', '_GLKVector3', '_GLKVector2', 'MortTable', 'CGPDFBox', 'AudioUnitParameterValueFromString', '__CFSocket', 'ALCdevice_struct', 'MIDINoteMessage', 'sfntFeatureHeader', 'CGRect', '__SCNetworkInterface', '__CFTree', 'MusicEventUserData', 'TrakTableData', 'MortContextualSubtable', '__CTRun', 'AudioUnitFrequencyResponseBin', 'MortChain', 'MorxInsertionSubtable', 'CGImageMetadata', 'gss_auth_identity', 'AudioUnitMIDIControlMapping', 'CAFChunkHeader', 'PropTable', 'CGPDFScanner', 'OpaqueMusicEventIterator', '__CFFileSecurity', 'AudioUnitNodeConnection', 'OpaqueMIDIDeviceList', 'ExtendedAudioFormatInfo', 'CGRectEdge', 'sfntFontDescriptor', '__CFRunLoopObserver', 'CGPatternTiling', 'MIDINotification', 'MorxLigatureSubtable', 'SFNTLookupSegment', 'MessageComposeResult', 'MIDIThruConnectionEndpoint', 'MusicDeviceStdNoteParams', 'opaqueCMSimpleQueue', 'ALCcontext_struct', 'OpaqueAudioQueue', 'PropLookupSingle', 'CGColor', 'AudioOutputUnitStartAtTimeParams', 'gss_name_t_desc_struct', 'CGFunctionCallbacks', 'CAFPacketTableHeader', 'AudioChannelDescription', 'sfntFeatureName', 'MorxContextualSubtable', 'CVSMPTETime', 'AudioValueRange', 'CGTextEncoding', 'AudioStreamBasicDescription', 'AUNodeRenderCallback', 'AudioPanningInfo', '__CFData', '__CFDate', 'KerxOrderedListEntry', '__CFAllocator', 'OpaqueJSPropertyNameArray', '__SCDynamicStore', 'OpaqueMIDIEntity', 'CFHostClientContext', 'CFNetServiceClientContext', 'AudioUnitPresetMAS_SettingData', 'opaqueCMBufferQueueTriggerToken', 'AudioUnitProperty', 'CAFRegionChunk', 'CGPDFString', '__CFWriteStream', '__CFAttributedString', '__CFStringTokenizer', 'JustWidthDeltaEntry', '__CFSet', 'sfntVariationAxis', '__CFNetDiagnostic', 'CAFOverviewSample', 'sfntCMapEncoding', 'CGVector', '__SCNetworkService', 'opaqueCMSampleBuffer', 'AUHostVersionIdentifier', 'AudioBalanceFade', 'sfntFontRunFeature', 'KerxCoordinateAction', 'sfntCMapSubHeader', 'CVPlanarPixelBufferInfo', 'AUNumVersion', '__CFTimeZone', 'AUSamplerInstrumentData', 'AUPreset', '__CTRunDelegate', 'OpaqueAudioQueueProcessingTap', 'KerxTableHeader', '_NSZone', 'OpaqueExtAudioFile', '__CFRunLoopSource', 'KerxAnchorPointAction', 'OpaqueJSString', 'AudioQueueParameterEvent', '__CFHTTPMessage', 'OpaqueCMClock', 'ScheduledAudioFileRegion', 'STEntryZero', 'gss_channel_bindings_struct', 'sfntVariationHeader', 'AUChannelInfo', 'UIOffset', 'GLKEffectPropertyPrv', 'KerxStateHeader', 'CGLineJoin', 'CGPDFDocument', '__CFBag', 'CFStreamErrorHTTPAuthentication', 'KernOrderedListHeader', '__SCNetworkSet', '__SecKey', 'MIDIObjectAddRemoveNotification', 'sfntDescriptorHeader', 'AudioUnitParameter', 'JustPCActionSubrecord', 'AudioComponentDescription', 'AudioUnitParameterValueName', 'AudioUnitParameterEvent', 'KerxControlPointAction', 'AudioTimeStamp', 'KernKerningPair', 'gss_buffer_set_desc_struct', 'MortFeatureEntry', 'FontVariation', 'CAFStringID', 'LcarCaretClassEntry', 'AudioUnitParameterStringFromValue', 'ACErrorCode', 'ALMXGlyphEntry', 'LtagTable', '__CTTypesetter', 'AuthorizationOpaqueRef', 'UIEdgeInsets', 'CGPathElement', 'CAFMarker', 'KernTableHeader', 'NoteParamsControlValue', 'SSLContext', 'gss_cred_id_t_desc_struct', 'AudioUnitParameterNameInfo', '__SecCertificate', 'CGDataConsumerCallbacks', 'CGInterpolationQuality', 'CGLineCap', 'MIDIControlTransform', 'BslnFormat1Part', 'CGPDFArray', '__SecPolicy', 'AudioConverterPrimeInfo', '__CTTextTab', '__CFNetServiceMonitor', 'AUInputSamplesInOutputCallbackStruct', '__CTFramesetter', 'CGPDFDataFormat', 'STHeader', 'CVPlanarPixelBufferInfo_YCbCrPlanar', 'MIDIValueMap', 'JustDirectionTable', '__SCBondStatus', 'SFNTLookupSegmentHeader', 'OpaqueCMMemoryPool', 'CGPathDrawingMode', 'CGFont', '__SCNetworkReachability', 'AudioClassDescription', 'CGPoint', 'CAFStrings', '__CFNetServiceBrowser', 'opaqueMTAudioProcessingTap', 'sfntNameRecord', 'CGPDFPage', 'CGLayer', 'ComponentInstanceRecord', 'CAFInfoStrings', 'HostCallbackInfo', 'MusicDeviceNoteParams', 'KernIndexArrayHeader', 'CVPlanarPixelBufferInfo_YCbCrBiPlanar', 'MusicTrackLoopInfo', 'opaqueCMFormatDescription', 'STClassTable', 'sfntDirectoryEntry', 'OpaqueCMTimebase', 'CGDataProviderDirectCallbacks', 'MIDIPacketList', 'CAFOverviewChunk', 'MIDIPacket', 'ScheduledAudioSlice', 'CGDataProviderSequentialCallbacks', 'AudioBuffer', 'MorxRearrangementSubtable', 'CGPatternCallbacks', 'AUDistanceAttenuationData', 'MIDIIOErrorNotification', 'CGPDFContentStream', 'IUnknownVTbl', 'MIDITransform', 'MortInsertionSubtable', 'CABarBeatTime', 'AudioBufferList', 'KerxSubtableHeader', '__CVBuffer', 'AURenderCallbackStruct', 'STXEntryZero', 'JustPCDuctilityAction', 'OpaqueAudioQueueTimeline', 'OpaqueMIDIClient', '__CFPlugInInstance', 'AudioQueueBuffer', '__CFFileDescriptor', 'AudioUnitConnection', '_GKTurnBasedExchangeStatus', 'LcarCaretTable', 'CVPlanarComponentInfo', 'JustWidthDeltaGroup', 'OpaqueAudioComponent', 'ParameterEvent', '__CVPixelBufferPool', '__CTFont', 'OpaqueJSClass', 'CGColorSpace', 'CGSize', 'AUDependentParameter', 'MIDIDriverInterface', 'gss_krb5_rfc1964_keydata', '__CFDateFormatter', 'LtagStringRange', 'CFNetServiceMonitorType', 'gss_iov_buffer_desc_struct', 'AUPresetEvent', 'CFNetServicesError', 'KernOrderedListEntry', '__CFLocale', 'gss_OID_desc_struct', 'AudioUnitPresetMAS_Settings', 'AudioFileMarker', 'JustPCConditionalAddAction', 'BslnFormat3Part', '__CFNotificationCenter', 'MortSwashSubtable', 'AUParameterMIDIMapping', 'OpaqueAudioConverter', 'MIDIRawData', 'CFNetDiagnosticStatusValues', 'sfntNameHeader', '__CFRunLoop', 'MFMailComposeResult', 'CATransform3D', 'OpbdSideValues', 'CAF_SMPTE_Time', 'JustPCAction', 'CGPathElementType', '__CFRunLoopTimer', '__CFError', 'AudioFormatListItem', '__CFReadStream', 'AudioUnitExternalBuffer', 'AudioFileRegion', 'AudioValueTranslation', 'CGImageMetadataTag', 'CAFPeakChunk', 'AudioBytePacketTranslation', 'CFNetworkErrors', 'sfntCMapHeader', '__CFURLEnumerator', '__CFCalendar', '__CFMessagePort', 'STXHeader', 'CGPDFObjectType', 'SFNTLookupArrayHeader']) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index d9311558..51d58280 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -18,8 +18,8 @@ from __future__ import print_function LEXERS = { 'ABAPLexer': ('pygments.lexers.other', 'ABAP', ('abap',), ('*.abap',), ('text/x-abap',)), 'APLLexer': ('pygments.lexers.other', 'APL', ('apl',), ('*.apl',), ()), - 'ActionScript3Lexer': ('pygments.lexers.web', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')), - 'ActionScriptLexer': ('pygments.lexers.web', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')), + 'ActionScript3Lexer': ('pygments.lexers.web', 'ActionScript 3', ('as3', 'actionscript3'), ('*.as',), ('application/x-actionscript3', 'text/x-actionscript3', 'text/actionscript3')), + 'ActionScriptLexer': ('pygments.lexers.web', 'ActionScript', ('as', 'actionscript'), ('*.as',), ('application/x-actionscript', 'text/x-actionscript', 'text/actionscript')), 'AdaLexer': ('pygments.lexers.compiled', 'Ada', ('ada', 'ada95ada2005'), ('*.adb', '*.ads', '*.ada'), ('text/x-ada',)), 'AgdaLexer': ('pygments.lexers.functional', 'Agda', ('agda',), ('*.agda',), ('text/x-agda',)), 'AmbientTalkLexer': ('pygments.lexers.other', 'AmbientTalk', ('at', 'ambienttalk', 'ambienttalk/2'), ('*.at',), ('text/x-ambienttalk',)), @@ -60,6 +60,7 @@ LEXERS = { 'CbmBasicV2Lexer': ('pygments.lexers.other', 'CBM BASIC V2', ('cbmbas',), ('*.bas',), ()), 'CeylonLexer': ('pygments.lexers.jvm', 'Ceylon', ('ceylon',), ('*.ceylon',), ('text/x-ceylon',)), 'Cfengine3Lexer': ('pygments.lexers.other', 'CFEngine3', ('cfengine3', 'cf3'), ('*.cf',), ()), + 'ChaiscriptLexer': ('pygments.lexers.agile', 'ChaiScript', ('chai', 'chaiscript'), ('*.chai',), ('text/x-chaiscript', 'application/x-chaiscript')), 'ChapelLexer': ('pygments.lexers.compiled', 'Chapel', ('chapel', 'chpl'), ('*.chpl',), ()), 'CheetahHtmlLexer': ('pygments.lexers.templates', 'HTML+Cheetah', ('html+cheetah', 'html+spitfire', 'htmlcheetah'), (), ('text/html+cheetah', 'text/html+spitfire')), 'CheetahJavascriptLexer': ('pygments.lexers.templates', 'JavaScript+Cheetah', ('js+cheetah', 'javascript+cheetah', 'js+spitfire', 'javascript+spitfire'), (), ('application/x-javascript+cheetah', 'text/x-javascript+cheetah', 'text/javascript+cheetah', 'application/x-javascript+spitfire', 'text/x-javascript+spitfire', 'text/javascript+spitfire')), @@ -80,6 +81,7 @@ LEXERS = { 'CppLexer': ('pygments.lexers.compiled', 'C++', ('cpp', 'c++'), ('*.cpp', '*.hpp', '*.c++', '*.h++', '*.cc', '*.hh', '*.cxx', '*.hxx', '*.C', '*.H', '*.cp', '*.CPP'), ('text/x-c++hdr', 'text/x-c++src')), 'CppObjdumpLexer': ('pygments.lexers.asm', 'cpp-objdump', ('cpp-objdump', 'c++-objdumb', 'cxx-objdump'), ('*.cpp-objdump', '*.c++-objdump', '*.cxx-objdump'), ('text/x-cpp-objdump',)), 'CrocLexer': ('pygments.lexers.agile', 'Croc', ('croc',), ('*.croc',), ('text/x-crocsrc',)), + 'CryptolLexer': ('pygments.lexers.functional', 'Cryptol', ('cryptol', 'cry'), ('*.cry',), ('text/x-cryptol',)), 'CssDjangoLexer': ('pygments.lexers.templates', 'CSS+Django/Jinja', ('css+django', 'css+jinja'), (), ('text/css+django', 'text/css+jinja')), 'CssErbLexer': ('pygments.lexers.templates', 'CSS+Ruby', ('css+erb', 'css+ruby'), (), ('text/css+ruby',)), 'CssGenshiLexer': ('pygments.lexers.templates', 'CSS+Genshi Text', ('css+genshitext', 'css+genshi'), (), ('text/css+genshi',)), @@ -87,6 +89,7 @@ LEXERS = { 'CssPhpLexer': ('pygments.lexers.templates', 'CSS+PHP', ('css+php',), (), ('text/css+php',)), 'CssSmartyLexer': ('pygments.lexers.templates', 'CSS+Smarty', ('css+smarty',), (), ('text/css+smarty',)), 'CudaLexer': ('pygments.lexers.compiled', 'CUDA', ('cuda', 'cu'), ('*.cu', '*.cuh'), ('text/x-cuda',)), + 'CypherLexer': ('pygments.lexers.graph', 'Cypher', ('cypher',), ('*.cyp', '*.cypher'), ()), 'CythonLexer': ('pygments.lexers.compiled', 'Cython', ('cython', 'pyx', 'pyrex'), ('*.pyx', '*.pxd', '*.pxi'), ('text/x-cython', 'application/x-cython')), 'DLexer': ('pygments.lexers.compiled', 'D', ('d',), ('*.d', '*.di'), ('text/x-dsrc',)), 'DObjdumpLexer': ('pygments.lexers.asm', 'd-objdump', ('d-objdump',), ('*.d-objdump',), ('text/x-d-objdump',)), @@ -97,6 +100,7 @@ LEXERS = { 'DgLexer': ('pygments.lexers.agile', 'dg', ('dg',), ('*.dg',), ('text/x-dg',)), 'DiffLexer': ('pygments.lexers.text', 'Diff', ('diff', 'udiff'), ('*.diff', '*.patch'), ('text/x-diff', 'text/x-patch')), 'DjangoLexer': ('pygments.lexers.templates', 'Django/Jinja', ('django', 'jinja'), (), ('application/x-django-templating', 'application/x-jinja')), + 'DockerLexer': ('pygments.lexers.text', 'Docker', ('docker', 'dockerfile'), ('Dockerfile', '*.docker'), ('text/x-dockerfile-config',)), 'DtdLexer': ('pygments.lexers.web', 'DTD', ('dtd',), ('*.dtd',), ('application/xml-dtd',)), 'DuelLexer': ('pygments.lexers.web', 'Duel', ('duel', 'jbst', 'jsonml+bst'), ('*.duel', '*.jbst'), ('text/x-duel', 'text/x-jbst')), 'DylanConsoleLexer': ('pygments.lexers.compiled', 'Dylan session', ('dylan-console', 'dylan-repl'), ('*.dylan-console',), ('text/x-dylan-console',)), @@ -162,6 +166,7 @@ LEXERS = { 'IrcLogsLexer': ('pygments.lexers.text', 'IRC logs', ('irc',), ('*.weechatlog',), ('text/x-irclog',)), 'JadeLexer': ('pygments.lexers.web', 'Jade', ('jade',), ('*.jade',), ('text/x-jade',)), 'JagsLexer': ('pygments.lexers.math', 'JAGS', ('jags',), ('*.jag', '*.bug'), ()), + 'JasminLexer': ('pygments.lexers.jvm', 'Jasmin', ('jasmin', 'jasminxt'), ('*.j',), ()), 'JavaLexer': ('pygments.lexers.jvm', 'Java', ('java',), ('*.java',), ('text/x-java',)), 'JavascriptDjangoLexer': ('pygments.lexers.templates', 'JavaScript+Django/Jinja', ('js+django', 'javascript+django', 'js+jinja', 'javascript+jinja'), (), ('application/x-javascript+django', 'application/x-javascript+jinja', 'text/x-javascript+django', 'text/x-javascript+jinja', 'text/javascript+django', 'text/javascript+jinja')), 'JavascriptErbLexer': ('pygments.lexers.templates', 'JavaScript+Ruby', ('js+erb', 'javascript+erb', 'js+ruby', 'javascript+ruby'), (), ('application/x-javascript+ruby', 'text/x-javascript+ruby', 'text/javascript+ruby')), @@ -186,6 +191,7 @@ LEXERS = { 'LighttpdConfLexer': ('pygments.lexers.text', 'Lighttpd configuration file', ('lighty', 'lighttpd'), (), ('text/x-lighttpd-conf',)), 'LimboLexer': ('pygments.lexers.inferno', 'Limbo', ('limbo',), ('*.b',), ('text/limbo',)), 'LiterateAgdaLexer': ('pygments.lexers.functional', 'Literate Agda', ('lagda', 'literate-agda'), ('*.lagda',), ('text/x-literate-agda',)), + 'LiterateCryptolLexer': ('pygments.lexers.functional', 'Literate Cryptol', ('lcry', 'literate-cryptol', 'lcryptol'), ('*.lcry',), ('text/x-literate-cryptol',)), 'LiterateHaskellLexer': ('pygments.lexers.functional', 'Literate Haskell', ('lhs', 'literate-haskell', 'lhaskell'), ('*.lhs',), ('text/x-literate-haskell',)), 'LiterateIdrisLexer': ('pygments.lexers.functional', 'Literate Idris', ('lidr', 'literate-idris', 'lidris'), ('*.lidr',), ('text/x-literate-idris',)), 'LiveScriptLexer': ('pygments.lexers.web', 'LiveScript', ('live-script', 'livescript'), ('*.ls',), ('text/livescript',)), @@ -242,6 +248,7 @@ LEXERS = { 'OocLexer': ('pygments.lexers.compiled', 'Ooc', ('ooc',), ('*.ooc',), ('text/x-ooc',)), 'OpaLexer': ('pygments.lexers.functional', 'Opa', ('opa',), ('*.opa',), ('text/x-opa',)), 'OpenEdgeLexer': ('pygments.lexers.other', 'OpenEdge ABL', ('openedge', 'abl', 'progress'), ('*.p', '*.cls'), ('text/x-openedge', 'application/x-openedge')), + 'PanLexer': ('pygments.lexers.other', 'Pan', ('pan',), ('*.pan',), ()), 'PawnLexer': ('pygments.lexers.other', 'Pawn', ('pawn',), ('*.p', '*.pwn', '*.inc'), ('text/x-pawn',)), 'Perl6Lexer': ('pygments.lexers.agile', 'Perl6', ('perl6', 'pl6'), ('*.pl', '*.pm', '*.nqp', '*.p6', '*.6pl', '*.p6l', '*.pl6', '*.6pm', '*.p6m', '*.pm6', '*.t'), ('text/x-perl6', 'application/x-perl6')), 'PerlLexer': ('pygments.lexers.agile', 'Perl', ('perl', 'pl'), ('*.pl', '*.pm', '*.t'), ('text/x-perl', 'application/x-perl')), @@ -264,10 +271,11 @@ LEXERS = { 'PythonConsoleLexer': ('pygments.lexers.agile', 'Python console session', ('pycon',), (), ('text/x-python-doctest',)), 'PythonLexer': ('pygments.lexers.agile', 'Python', ('python', 'py', 'sage'), ('*.py', '*.pyw', '*.sc', 'SConstruct', 'SConscript', '*.tac', '*.sage'), ('text/x-python', 'application/x-python')), 'PythonTracebackLexer': ('pygments.lexers.agile', 'Python Traceback', ('pytb',), ('*.pytb',), ('text/x-python-traceback',)), + 'QBasicLexer': ('pygments.lexers.qbasic', 'QBasic', ('qbasic', 'basic'), ('*.BAS', '*.bas'), ('text/basic',)), 'QmlLexer': ('pygments.lexers.web', 'QML', ('qml',), ('*.qml',), ('application/x-qml',)), 'RConsoleLexer': ('pygments.lexers.math', 'RConsole', ('rconsole', 'rout'), ('*.Rout',), ()), 'RPMSpecLexer': ('pygments.lexers.other', 'RPMSpec', ('spec',), ('*.spec',), ('text/x-rpm-spec',)), - 'RacketLexer': ('pygments.lexers.functional', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktl'), ('text/x-racket', 'application/x-racket')), + 'RacketLexer': ('pygments.lexers.functional', 'Racket', ('racket', 'rkt'), ('*.rkt', '*.rktd', '*.rktl'), ('text/x-racket', 'application/x-racket')), 'RagelCLexer': ('pygments.lexers.parsers', 'Ragel in C Host', ('ragel-c',), ('*.rl',), ()), 'RagelCppLexer': ('pygments.lexers.parsers', 'Ragel in CPP Host', ('ragel-cpp',), ('*.rl',), ()), 'RagelDLexer': ('pygments.lexers.parsers', 'Ragel in D Host', ('ragel-d',), ('*.rl',), ()), @@ -278,13 +286,15 @@ LEXERS = { 'RagelRubyLexer': ('pygments.lexers.parsers', 'Ragel in Ruby Host', ('ragel-ruby', 'ragel-rb'), ('*.rl',), ()), 'RawTokenLexer': ('pygments.lexers.special', 'Raw token data', ('raw',), (), ('application/x-pygments-tokens',)), 'RdLexer': ('pygments.lexers.math', 'Rd', ('rd',), ('*.Rd',), ('text/x-r-doc',)), - 'RebolLexer': ('pygments.lexers.other', 'REBOL', ('rebol',), ('*.r', '*.r3'), ('text/x-rebol',)), + 'RebolLexer': ('pygments.lexers.other', 'REBOL', ('rebol',), ('*.r', '*.r3', '*.reb'), ('text/x-rebol',)), + 'RedLexer': ('pygments.lexers.other', 'Red', ('red', 'red/system'), ('*.red', '*.reds'), ('text/x-red', 'text/x-red-system')), 'RedcodeLexer': ('pygments.lexers.other', 'Redcode', ('redcode',), ('*.cw',), ()), 'RegeditLexer': ('pygments.lexers.text', 'reg', ('registry',), ('*.reg',), ('text/x-windows-registry',)), 'RexxLexer': ('pygments.lexers.other', 'Rexx', ('rexx', 'arexx'), ('*.rexx', '*.rex', '*.rx', '*.arexx'), ('text/x-rexx',)), 'RhtmlLexer': ('pygments.lexers.templates', 'RHTML', ('rhtml', 'html+erb', 'html+ruby'), ('*.rhtml',), ('text/html+ruby',)), 'RobotFrameworkLexer': ('pygments.lexers.other', 'RobotFramework', ('robotframework',), ('*.txt', '*.robot'), ('text/x-robotframework',)), 'RqlLexer': ('pygments.lexers.sql', 'RQL', ('rql',), ('*.rql',), ('text/x-rql',)), + 'RslLexer': ('pygments.lexers.other', 'RSL', ('rsl',), ('*.rsl',), ('text/rsl',)), 'RstLexer': ('pygments.lexers.text', 'reStructuredText', ('rst', 'rest', 'restructuredtext'), ('*.rst', '*.rest'), ('text/x-rst', 'text/prs.fallenstein.rst')), 'RubyConsoleLexer': ('pygments.lexers.agile', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.agile', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby'), ('text/x-ruby', 'application/x-ruby')), diff --git a/pygments/lexers/_stan_builtins.py b/pygments/lexers/_stan_builtins.py index fa45738d..583b44a8 100644 --- a/pygments/lexers/_stan_builtins.py +++ b/pygments/lexers/_stan_builtins.py @@ -4,7 +4,7 @@ ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ This file contains the names of functions for Stan used by - `pygments.lexers.math.StanLexer`. + ``pygments.lexers.math.StanLexer``. These builtins are from Stan language v2.2.0. :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. @@ -91,9 +91,11 @@ FUNCTIONS = [ 'Phi', 'diag_post_multiply', 'diag_pre_multiply', 'diagonal', + 'digamma', 'dims', 'dirichlet_log', 'dirichlet_rng', + 'distance', 'dot_product', 'dot_self', 'double_exponential_ccdf_log', @@ -170,7 +172,6 @@ FUNCTIONS = [ 'Phi', 'lgamma', 'lkj_corr_log', 'lkj_corr_rng', - 'lkj_cov_log', 'lmgamma', 'log', 'log10', @@ -285,6 +286,7 @@ FUNCTIONS = [ 'Phi', 'sqrt', 'sqrt2', 'square', + 'squared_distance', 'step', 'student_t_ccdf_log', 'student_t_cdf', @@ -303,6 +305,7 @@ FUNCTIONS = [ 'Phi', 'trace', 'trace_gen_quad_form', 'trace_quad_form', + 'trigamma', 'trunc', 'uniform_ccdf_log', 'uniform_cdf', @@ -324,7 +327,6 @@ DISTRIBUTIONS = [ 'bernoulli', 'beta', 'beta_binomial', 'binomial', - 'binomial_coefficient', 'binomial_logit', 'categorical', 'categorical_logit', @@ -342,7 +344,6 @@ DISTRIBUTIONS = [ 'bernoulli', 'inv_gamma', 'inv_wishart', 'lkj_corr', - 'lkj_cov', 'logistic', 'lognormal', 'multi_normal', @@ -350,7 +351,6 @@ DISTRIBUTIONS = [ 'bernoulli', 'multi_normal_prec', 'multi_student_t', 'multinomial', - 'multiply', 'neg_binomial', 'normal', 'ordered_logistic', @@ -451,3 +451,4 @@ RESERVED = [ 'alignas', 'wchar_t', 'xor', 'xor_eq'] + diff --git a/pygments/lexers/agile.py b/pygments/lexers/agile.py index 0a780a3e..cd105126 100644 --- a/pygments/lexers/agile.py +++ b/pygments/lexers/agile.py @@ -23,7 +23,8 @@ __all__ = ['PythonLexer', 'PythonConsoleLexer', 'PythonTracebackLexer', 'Python3Lexer', 'Python3TracebackLexer', 'RubyLexer', 'RubyConsoleLexer', 'PerlLexer', 'LuaLexer', 'MoonScriptLexer', 'CrocLexer', 'MiniDLexer', 'IoLexer', 'TclLexer', 'FactorLexer', - 'FancyLexer', 'DgLexer', 'Perl6Lexer', 'HyLexer'] + 'FancyLexer', 'DgLexer', 'Perl6Lexer', 'HyLexer', + 'ChaiscriptLexer'] # b/w compatibility from pygments.lexers.functional import SchemeLexer @@ -118,20 +119,20 @@ class PythonLexer(RegexLexer): ('`.*?`', String.Backtick), ], 'name': [ - (r'@[a-zA-Z0-9_.]+', Name.Decorator), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'@[\w.]+', Name.Decorator), + ('[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop') + ('[a-zA-Z_]\w*', Name.Function, '#pop') ], 'classname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(?:[ \t]|\\\n)+', Text), (r'as\b', Keyword.Namespace), (r',', Operator), - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_][\w.]*', Name.Namespace), (r'', Text, '#pop') # all else: go back ], 'fromimport': [ @@ -141,7 +142,7 @@ class PythonLexer(RegexLexer): # never be a module name (r'None\b', Name.Builtin.Pseudo, '#pop'), # sadly, in "raise x from y" y will be highlighted as namespace too - (r'[a-zA-Z_.][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_.][\w.]*', Name.Namespace), # anything else here also means "raise x from y" and is therefore # not an error (r'', Text, '#pop'), @@ -151,7 +152,7 @@ class PythonLexer(RegexLexer): r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) ], 'strings': [ - (r'%(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), (r'[^\\\'"%\n]+', String), # quotes, percents and backslashes must be parsed one at a time @@ -254,7 +255,7 @@ class Python3Lexer(RegexLexer): ] tokens['backtick'] = [] tokens['name'] = [ - (r'@[a-zA-Z0-9_]+', Name.Decorator), + (r'@\w+', Name.Decorator), (uni_name, Name), ] tokens['funcname'] = [ @@ -405,7 +406,7 @@ class PythonTracebackLexer(RegexLexer): bygroups(Text, Comment, Text)), # for doctests... (r'^([^:]+)(: )(.+)(\n)', bygroups(Generic.Error, Text, Name, Text), '#pop'), - (r'^([a-zA-Z_][a-zA-Z0-9_]*)(:?\n)', + (r'^([a-zA-Z_]\w*)(:?\n)', bygroups(Generic.Error, Text), '#pop') ], } @@ -444,7 +445,7 @@ class Python3TracebackLexer(RegexLexer): bygroups(Text, Comment, Text)), # for doctests... (r'^([^:]+)(: )(.+)(\n)', bygroups(Generic.Error, Text, Name, Text), '#pop'), - (r'^([a-zA-Z_][a-zA-Z0-9_]*)(:?\n)', + (r'^([a-zA-Z_]\w*)(:?\n)', bygroups(Generic.Error, Text), '#pop') ], } @@ -534,7 +535,7 @@ class RubyLexer(ExtendedRegexLexer): (r":'(\\\\|\\'|[^'])*'", String.Symbol), (r"'(\\\\|\\'|[^'])*'", String.Single), (r':"', String.Symbol, 'simple-sym'), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(:)(?!:)', + (r'([a-zA-Z_]\w*)(:)(?!:)', bygroups(String.Symbol, Punctuation)), # Since Ruby 1.9 (r'"', String.Double, 'simple-string'), (r'(?<!\.)`', String.Backtick, 'simple-backtick'), @@ -620,8 +621,8 @@ class RubyLexer(ExtendedRegexLexer): r'rescue|raise|retry|return|super|then|undef|unless|until|when|' r'while|yield)\b', Keyword), # start of function, class and module names - (r'(module)(\s+)([a-zA-Z_][a-zA-Z0-9_]*' - r'(?:::[a-zA-Z_][a-zA-Z0-9_]*)*)', + (r'(module)(\s+)([a-zA-Z_]\w*' + r'(?:::[a-zA-Z_]\w*)*)', bygroups(Keyword, Text, Name.Namespace)), (r'(def)(\s+)', bygroups(Keyword, Text), 'funcname'), (r'def(?=[*%&^`~+-/\[<>=])', Keyword, 'funcname'), @@ -712,9 +713,9 @@ class RubyLexer(ExtendedRegexLexer): (r'([\d]+(?:_\d+)*)(\s*)([/?])?', bygroups(Number.Integer, Text, Operator)), # Names - (r'@@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Class), - (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Instance), - (r'\$[a-zA-Z0-9_]+', Name.Variable.Global), + (r'@@[a-zA-Z_]\w*', Name.Variable.Class), + (r'@[a-zA-Z_]\w*', Name.Variable.Instance), + (r'\$\w+', Name.Variable.Global), (r'\$[!@&`\'+~=/\\,;.<>_*$?:"]', Name.Variable.Global), (r'\$-[0adFiIlpvw]', Name.Variable.Global), (r'::', Operator), @@ -724,7 +725,7 @@ class RubyLexer(ExtendedRegexLexer): r'(\\([\\abefnrstv#"\']|x[a-fA-F0-9]{1,2}|[0-7]{1,3})|\S)' r'(?!\w)', String.Char), - (r'[A-Z][a-zA-Z0-9_]+', Name.Constant), + (r'[A-Z]\w+', Name.Constant), # this is needed because ruby attributes can look # like keywords (class) or like this: ` ?!? (r'(\.|::)([a-zA-Z_]\w*[\!\?]?|[*%&^`~+-/\[<>=])', @@ -738,7 +739,7 @@ class RubyLexer(ExtendedRegexLexer): ], 'funcname': [ (r'\(', Punctuation, 'defexpr'), - (r'(?:([a-zA-Z_][a-zA-Z0-9_]*)(\.))?' + (r'(?:([a-zA-Z_]\w*)(\.))?' r'([a-zA-Z_]\w*[\!\?]?|\*\*?|[-+]@?|' r'[/%&|^`~]|\[\]=?|<<|>>|<=?>|>=?|===?)', bygroups(Name.Class, Operator, Name.Function), '#pop'), @@ -761,8 +762,8 @@ class RubyLexer(ExtendedRegexLexer): ], 'string-intp': [ (r'#{', String.Interpol, 'in-intp'), - (r'#@@?[a-zA-Z_][a-zA-Z0-9_]*', String.Interpol), - (r'#\$[a-zA-Z_][a-zA-Z0-9_]*', String.Interpol) + (r'#@@?[a-zA-Z_]\w*', String.Interpol), + (r'#\$[a-zA-Z_]\w*', String.Interpol) ], 'string-intp-escaped': [ include('string-intp'), @@ -813,7 +814,7 @@ class RubyConsoleLexer(Lexer): aliases = ['rbcon', 'irb'] mimetypes = ['text/x-ruby-shellsession'] - _prompt_re = re.compile('irb\([a-zA-Z_][a-zA-Z0-9_]*\):\d{3}:\d+[>*"\'] ' + _prompt_re = re.compile('irb\([a-zA-Z_]\w*\):\d{3}:\d+[>*"\'] ' '|>> |\?> ') def get_tokens_unprocessed(self, text): @@ -874,7 +875,7 @@ class PerlLexer(RegexLexer): (r'(case|continue|do|else|elsif|for|foreach|if|last|my|' r'next|our|redo|reset|then|unless|until|while|use|' r'print|new|BEGIN|CHECK|INIT|END|return)\b', Keyword), - (r'(format)(\s+)([a-zA-Z0-9_]+)(\s*)(=)(\s*\n)', + (r'(format)(\s+)(\w+)(\s*)(=)(\s*\n)', bygroups(Keyword, Text, Name, Text, Punctuation, Text), 'format'), (r'(eq|lt|gt|le|ge|ne|not|and|or|cmp)\b', Operator.Word), # common delimiters @@ -927,7 +928,7 @@ class PerlLexer(RegexLexer): r'utime|values|vec|wait|waitpid|wantarray|warn|write' r')\b', Name.Builtin), (r'((__(DATA|DIE|WARN)__)|(STD(IN|OUT|ERR)))\b', Name.Builtin.Pseudo), - (r'<<([\'"]?)([a-zA-Z_][a-zA-Z0-9_]*)\1;?\n.*?\n\2\n', String), + (r'<<([\'"]?)([a-zA-Z_]\w*)\1;?\n.*?\n\2\n', String), (r'__END__', Comment.Preproc, 'end-part'), (r'\$\^[ADEFHILMOPSTWX]', Name.Variable.Global), (r"\$[\\\"\[\]'&`+*.,;=%~?@$!<>(^|/-](?!\w)", Name.Variable.Global), @@ -965,14 +966,14 @@ class PerlLexer(RegexLexer): (r'\s+', Text), (r'\{', Punctuation, '#pop'), # hash syntax? (r'\)|,', Punctuation, '#pop'), # argument specifier - (r'[a-zA-Z0-9_]+::', Name.Namespace), - (r'[a-zA-Z0-9_:]+', Name.Variable, '#pop'), + (r'\w+::', Name.Namespace), + (r'[\w:]+', Name.Variable, '#pop'), ], 'name': [ - (r'[a-zA-Z0-9_]+::', Name.Namespace), - (r'[a-zA-Z0-9_:]+', Name, '#pop'), - (r'[A-Z_]+(?=[^a-zA-Z0-9_])', Name.Constant, '#pop'), - (r'(?=[^a-zA-Z0-9_])', Text, '#pop'), + (r'\w+::', Name.Namespace), + (r'[\w:]+', Name, '#pop'), + (r'[A-Z_]+(?=\W)', Name.Constant, '#pop'), + (r'(?=\W)', Text, '#pop'), ], 'modulename': [ (r'[a-zA-Z_]\w*', Name.Namespace, '#pop') @@ -1084,7 +1085,7 @@ class LuaLexer(RegexLexer): (r'(function)\b', Keyword, 'funcname'), - (r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?', Name), + (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name), ("'", String.Single, combined('stringescape', 'sqs')), ('"', String.Double, combined('stringescape', 'dqs')) @@ -1092,7 +1093,7 @@ class LuaLexer(RegexLexer): 'funcname': [ (r'\s+', Text), - ('(?:([A-Za-z_][A-Za-z0-9_]*)(\.))?([A-Za-z_][A-Za-z0-9_]*)', + ('(?:([A-Za-z_]\w*)(\.))?([A-Za-z_]\w*)', bygroups(Name.Class, Punctuation, Name.Function), '#pop'), # inline function ('\(', Punctuation, '#pop'), @@ -1175,20 +1176,20 @@ class MoonScriptLexer(LuaLexer): (r'[^\S\n]+', Text), (r'(?s)\[(=*)\[.*?\]\1\]', String), (r'(->|=>)', Name.Function), - (r':[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r':[a-zA-Z_]\w*', Name.Variable), (r'(==|!=|~=|<=|>=|\.\.\.|\.\.|[=+\-*/%^<>#!.\\:])', Operator), (r'[;,]', Punctuation), (r'[\[\]\{\}\(\)]', Keyword.Type), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Variable), + (r'[a-zA-Z_]\w*:', Name.Variable), (r"(class|extends|if|then|super|do|with|import|export|" r"while|elseif|return|for|in|from|when|using|else|" r"and|or|not|switch|break)\b", Keyword), (r'(true|false|nil)\b', Keyword.Constant), (r'(and|or|not)\b', Operator.Word), (r'(self)\b', Name.Builtin.Pseudo), - (r'@@?([a-zA-Z_][a-zA-Z0-9_]*)?', Name.Variable.Class), + (r'@@?([a-zA-Z_]\w*)?', Name.Variable.Class), (r'[A-Z]\w*', Name.Class), # proper name - (r'[A-Za-z_][A-Za-z0-9_]*(\.[A-Za-z_][A-Za-z0-9_]*)?', Name), + (r'[A-Za-z_]\w*(\.[A-Za-z_]\w*)?', Name), ("'", String.Single, combined('stringescape', 'sqs')), ('"', String.Double, combined('stringescape', 'dqs')) ], @@ -1319,7 +1320,7 @@ class IoLexer(RegexLexer): # names (r'(Object|list|List|Map|args|Sequence|Coroutine|File)\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), # numbers (r'(\d+\.?\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), (r'\d+', Number.Integer) @@ -1720,7 +1721,7 @@ class FactorLexer(RegexLexer): (r'"""\s+(?:.|\n)*?\s+"""', String), (r'"(?:\\\\|\\"|[^"])*"', String), (r'\S+"\s+(?:\\\\|\\"|[^"])*"', String), - (r'CHAR:\s+(\\[\\abfnrstv]|[^\\]\S+)\s', String.Char), + (r'CHAR:\s+(?:\\[\\abfnrstv]|[^\\]\S*)\s', String.Char), # comments (r'!\s+.*$', Comment), @@ -1728,21 +1729,21 @@ class FactorLexer(RegexLexer): (r'/\*\s+(?:.|\n)*?\s\*/\s', Comment), # boolean constants - (r'(t|f)\s', Name.Constant), + (r'[tf]\s', Name.Constant), # symbols and literals (r'[\\$]\s+\S+', Name.Constant), (r'M\\\s+\S+\s+\S+', Name.Constant), # numbers - (r'[+-]?([\d,]*\d)?\.(\d([\d,]*\d)?)?([eE][+-]?\d+)?\s', Number), - (r'[+-]?\d([\d,]*\d)?([eE][+-]?\d+)?\s', Number), - (r'0x[a-fA-F\d]([a-fA-F\d,]*[a-fA-F\d])?(p\d([\d,]*\d)?)?\s', Number), - (r'NAN:\s+[a-fA-F\d]([a-fA-F\d,]*[a-fA-F\d])?(p\d([\d,]*\d)?)?\s', Number), + (r'[+-]?(?:[\d,]*\d)?\.(?:\d([\d,]*\d)?)?(?:[eE][+-]?\d+)?\s', Number), + (r'[+-]?\d(?:[\d,]*\d)?(?:[eE][+-]?\d+)?\s', Number), + (r'0x[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number), + (r'NAN:\s+[a-fA-F\d](?:[a-fA-F\d,]*[a-fA-F\d])?(?:p\d([\d,]*\d)?)?\s', Number), (r'0b[01]+\s', Number), (r'0o[0-7]+\s', Number), - (r'(\d([\d,]*\d)?)?\+\d([\d,]*\d)?/\d([\d,]*\d)?\s', Number), - (r'(\-\d([\d,]*\d)?)?\-\d([\d,]*\d)?/\d([\d,]*\d)?\s', Number), + (r'(?:\d([\d,]*\d)?)?\+\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number), + (r'(?:\-\d([\d,]*\d)?)?\-\d(?:[\d,]*\d)?/\d(?:[\d,]*\d)?\s', Number), # keywords (r'(?:deprecated|final|foldable|flushable|inline|recursive)\s', @@ -1856,14 +1857,14 @@ class FancyLexer(RegexLexer): r'FalseClass|Tuple|Symbol|Stack|Set|FancySpec|Method|Package|' r'Range)\b', Name.Builtin), # functions - (r'[a-zA-Z]([a-zA-Z0-9_]|[-+?!=*/^><%])*:', Name.Function), + (r'[a-zA-Z](\w|[-+?!=*/^><%])*:', Name.Function), # operators, must be below functions (r'[-+*/~,<>=&!?%^\[\]\.$]+', Operator), - ('[A-Z][a-zA-Z0-9_]*', Name.Constant), - ('@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Instance), - ('@@[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable.Class), + ('[A-Z]\w*', Name.Constant), + ('@[a-zA-Z_]\w*', Name.Variable.Instance), + ('@@[a-zA-Z_]\w*', Name.Variable.Class), ('@@?', Operator), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), # numbers - / checks are necessary to avoid mismarking regexes, # see comment in RubyLexer (r'(0[oO]?[0-7]+(?:_[0-7]+)*)(\s*)([/?])?', @@ -1948,7 +1949,7 @@ class DgLexer(RegexLexer): r'U[a-fA-F0-9]{8}|x[a-fA-F0-9]{2}|[0-7]{1,3})', String.Escape) ], 'string': [ - (r'%(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' + (r'%(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?' '[hlL]?[diouxXeEfFgGcrs%]', String.Interpol), (r'[^\\\'"%\n]+', String), # quotes, percents and backslashes must be parsed one at a time @@ -1986,7 +1987,7 @@ class Perl6Lexer(ExtendedRegexLexer): mimetypes = ['text/x-perl6', 'application/x-perl6'] flags = re.MULTILINE | re.DOTALL | re.UNICODE - PERL6_IDENTIFIER_RANGE = "['a-zA-Z0-9_:-]" # if you alter this, search for a copy made of it below + PERL6_IDENTIFIER_RANGE = "['\w:-]" PERL6_KEYWORDS = ( 'BEGIN', 'CATCH', 'CHECK', 'CONTROL', 'END', 'ENTER', 'FIRST', 'INIT', @@ -2077,62 +2078,80 @@ class Perl6Lexer(ExtendedRegexLexer): # Perl 6 has a *lot* of possible bracketing characters # this list was lifted from STD.pm6 (https://github.com/perl6/std) PERL6_BRACKETS = { - u'\u0028' : u'\u0029', u'\u003c' : u'\u003e', u'\u005b' : u'\u005d', u'\u007b' : u'\u007d', - u'\u00ab' : u'\u00bb', u'\u0f3a' : u'\u0f3b', u'\u0f3c' : u'\u0f3d', u'\u169b' : u'\u169c', - u'\u2018' : u'\u2019', u'\u201a' : u'\u2019', u'\u201b' : u'\u2019', u'\u201c' : u'\u201d', - u'\u201e' : u'\u201d', u'\u201f' : u'\u201d', u'\u2039' : u'\u203a', u'\u2045' : u'\u2046', - u'\u207d' : u'\u207e', u'\u208d' : u'\u208e', u'\u2208' : u'\u220b', u'\u2209' : u'\u220c', - u'\u220a' : u'\u220d', u'\u2215' : u'\u29f5', u'\u223c' : u'\u223d', u'\u2243' : u'\u22cd', - u'\u2252' : u'\u2253', u'\u2254' : u'\u2255', u'\u2264' : u'\u2265', u'\u2266' : u'\u2267', - u'\u2268' : u'\u2269', u'\u226a' : u'\u226b', u'\u226e' : u'\u226f', u'\u2270' : u'\u2271', - u'\u2272' : u'\u2273', u'\u2274' : u'\u2275', u'\u2276' : u'\u2277', u'\u2278' : u'\u2279', - u'\u227a' : u'\u227b', u'\u227c' : u'\u227d', u'\u227e' : u'\u227f', u'\u2280' : u'\u2281', - u'\u2282' : u'\u2283', u'\u2284' : u'\u2285', u'\u2286' : u'\u2287', u'\u2288' : u'\u2289', - u'\u228a' : u'\u228b', u'\u228f' : u'\u2290', u'\u2291' : u'\u2292', u'\u2298' : u'\u29b8', - u'\u22a2' : u'\u22a3', u'\u22a6' : u'\u2ade', u'\u22a8' : u'\u2ae4', u'\u22a9' : u'\u2ae3', - u'\u22ab' : u'\u2ae5', u'\u22b0' : u'\u22b1', u'\u22b2' : u'\u22b3', u'\u22b4' : u'\u22b5', - u'\u22b6' : u'\u22b7', u'\u22c9' : u'\u22ca', u'\u22cb' : u'\u22cc', u'\u22d0' : u'\u22d1', - u'\u22d6' : u'\u22d7', u'\u22d8' : u'\u22d9', u'\u22da' : u'\u22db', u'\u22dc' : u'\u22dd', - u'\u22de' : u'\u22df', u'\u22e0' : u'\u22e1', u'\u22e2' : u'\u22e3', u'\u22e4' : u'\u22e5', - u'\u22e6' : u'\u22e7', u'\u22e8' : u'\u22e9', u'\u22ea' : u'\u22eb', u'\u22ec' : u'\u22ed', - u'\u22f0' : u'\u22f1', u'\u22f2' : u'\u22fa', u'\u22f3' : u'\u22fb', u'\u22f4' : u'\u22fc', - u'\u22f6' : u'\u22fd', u'\u22f7' : u'\u22fe', u'\u2308' : u'\u2309', u'\u230a' : u'\u230b', - u'\u2329' : u'\u232a', u'\u23b4' : u'\u23b5', u'\u2768' : u'\u2769', u'\u276a' : u'\u276b', - u'\u276c' : u'\u276d', u'\u276e' : u'\u276f', u'\u2770' : u'\u2771', u'\u2772' : u'\u2773', - u'\u2774' : u'\u2775', u'\u27c3' : u'\u27c4', u'\u27c5' : u'\u27c6', u'\u27d5' : u'\u27d6', - u'\u27dd' : u'\u27de', u'\u27e2' : u'\u27e3', u'\u27e4' : u'\u27e5', u'\u27e6' : u'\u27e7', - u'\u27e8' : u'\u27e9', u'\u27ea' : u'\u27eb', u'\u2983' : u'\u2984', u'\u2985' : u'\u2986', - u'\u2987' : u'\u2988', u'\u2989' : u'\u298a', u'\u298b' : u'\u298c', u'\u298d' : u'\u298e', - u'\u298f' : u'\u2990', u'\u2991' : u'\u2992', u'\u2993' : u'\u2994', u'\u2995' : u'\u2996', - u'\u2997' : u'\u2998', u'\u29c0' : u'\u29c1', u'\u29c4' : u'\u29c5', u'\u29cf' : u'\u29d0', - u'\u29d1' : u'\u29d2', u'\u29d4' : u'\u29d5', u'\u29d8' : u'\u29d9', u'\u29da' : u'\u29db', - u'\u29f8' : u'\u29f9', u'\u29fc' : u'\u29fd', u'\u2a2b' : u'\u2a2c', u'\u2a2d' : u'\u2a2e', - u'\u2a34' : u'\u2a35', u'\u2a3c' : u'\u2a3d', u'\u2a64' : u'\u2a65', u'\u2a79' : u'\u2a7a', - u'\u2a7d' : u'\u2a7e', u'\u2a7f' : u'\u2a80', u'\u2a81' : u'\u2a82', u'\u2a83' : u'\u2a84', - u'\u2a8b' : u'\u2a8c', u'\u2a91' : u'\u2a92', u'\u2a93' : u'\u2a94', u'\u2a95' : u'\u2a96', - u'\u2a97' : u'\u2a98', u'\u2a99' : u'\u2a9a', u'\u2a9b' : u'\u2a9c', u'\u2aa1' : u'\u2aa2', - u'\u2aa6' : u'\u2aa7', u'\u2aa8' : u'\u2aa9', u'\u2aaa' : u'\u2aab', u'\u2aac' : u'\u2aad', - u'\u2aaf' : u'\u2ab0', u'\u2ab3' : u'\u2ab4', u'\u2abb' : u'\u2abc', u'\u2abd' : u'\u2abe', - u'\u2abf' : u'\u2ac0', u'\u2ac1' : u'\u2ac2', u'\u2ac3' : u'\u2ac4', u'\u2ac5' : u'\u2ac6', - u'\u2acd' : u'\u2ace', u'\u2acf' : u'\u2ad0', u'\u2ad1' : u'\u2ad2', u'\u2ad3' : u'\u2ad4', - u'\u2ad5' : u'\u2ad6', u'\u2aec' : u'\u2aed', u'\u2af7' : u'\u2af8', u'\u2af9' : u'\u2afa', - u'\u2e02' : u'\u2e03', u'\u2e04' : u'\u2e05', u'\u2e09' : u'\u2e0a', u'\u2e0c' : u'\u2e0d', - u'\u2e1c' : u'\u2e1d', u'\u2e20' : u'\u2e21', u'\u3008' : u'\u3009', u'\u300a' : u'\u300b', - u'\u300c' : u'\u300d', u'\u300e' : u'\u300f', u'\u3010' : u'\u3011', u'\u3014' : u'\u3015', - u'\u3016' : u'\u3017', u'\u3018' : u'\u3019', u'\u301a' : u'\u301b', u'\u301d' : u'\u301e', - u'\ufd3e' : u'\ufd3f', u'\ufe17' : u'\ufe18', u'\ufe35' : u'\ufe36', u'\ufe37' : u'\ufe38', - u'\ufe39' : u'\ufe3a', u'\ufe3b' : u'\ufe3c', u'\ufe3d' : u'\ufe3e', u'\ufe3f' : u'\ufe40', - u'\ufe41' : u'\ufe42', u'\ufe43' : u'\ufe44', u'\ufe47' : u'\ufe48', u'\ufe59' : u'\ufe5a', - u'\ufe5b' : u'\ufe5c', u'\ufe5d' : u'\ufe5e', u'\uff08' : u'\uff09', u'\uff1c' : u'\uff1e', - u'\uff3b' : u'\uff3d', u'\uff5b' : u'\uff5d', u'\uff5f' : u'\uff60', u'\uff62' : u'\uff63', + u'\u0028' : u'\u0029', u'\u003c' : u'\u003e', u'\u005b' : u'\u005d', + u'\u007b' : u'\u007d', u'\u00ab' : u'\u00bb', u'\u0f3a' : u'\u0f3b', + u'\u0f3c' : u'\u0f3d', u'\u169b' : u'\u169c', u'\u2018' : u'\u2019', + u'\u201a' : u'\u2019', u'\u201b' : u'\u2019', u'\u201c' : u'\u201d', + u'\u201e' : u'\u201d', u'\u201f' : u'\u201d', u'\u2039' : u'\u203a', + u'\u2045' : u'\u2046', u'\u207d' : u'\u207e', u'\u208d' : u'\u208e', + u'\u2208' : u'\u220b', u'\u2209' : u'\u220c', u'\u220a' : u'\u220d', + u'\u2215' : u'\u29f5', u'\u223c' : u'\u223d', u'\u2243' : u'\u22cd', + u'\u2252' : u'\u2253', u'\u2254' : u'\u2255', u'\u2264' : u'\u2265', + u'\u2266' : u'\u2267', u'\u2268' : u'\u2269', u'\u226a' : u'\u226b', + u'\u226e' : u'\u226f', u'\u2270' : u'\u2271', u'\u2272' : u'\u2273', + u'\u2274' : u'\u2275', u'\u2276' : u'\u2277', u'\u2278' : u'\u2279', + u'\u227a' : u'\u227b', u'\u227c' : u'\u227d', u'\u227e' : u'\u227f', + u'\u2280' : u'\u2281', u'\u2282' : u'\u2283', u'\u2284' : u'\u2285', + u'\u2286' : u'\u2287', u'\u2288' : u'\u2289', u'\u228a' : u'\u228b', + u'\u228f' : u'\u2290', u'\u2291' : u'\u2292', u'\u2298' : u'\u29b8', + u'\u22a2' : u'\u22a3', u'\u22a6' : u'\u2ade', u'\u22a8' : u'\u2ae4', + u'\u22a9' : u'\u2ae3', u'\u22ab' : u'\u2ae5', u'\u22b0' : u'\u22b1', + u'\u22b2' : u'\u22b3', u'\u22b4' : u'\u22b5', u'\u22b6' : u'\u22b7', + u'\u22c9' : u'\u22ca', u'\u22cb' : u'\u22cc', u'\u22d0' : u'\u22d1', + u'\u22d6' : u'\u22d7', u'\u22d8' : u'\u22d9', u'\u22da' : u'\u22db', + u'\u22dc' : u'\u22dd', u'\u22de' : u'\u22df', u'\u22e0' : u'\u22e1', + u'\u22e2' : u'\u22e3', u'\u22e4' : u'\u22e5', u'\u22e6' : u'\u22e7', + u'\u22e8' : u'\u22e9', u'\u22ea' : u'\u22eb', u'\u22ec' : u'\u22ed', + u'\u22f0' : u'\u22f1', u'\u22f2' : u'\u22fa', u'\u22f3' : u'\u22fb', + u'\u22f4' : u'\u22fc', u'\u22f6' : u'\u22fd', u'\u22f7' : u'\u22fe', + u'\u2308' : u'\u2309', u'\u230a' : u'\u230b', u'\u2329' : u'\u232a', + u'\u23b4' : u'\u23b5', u'\u2768' : u'\u2769', u'\u276a' : u'\u276b', + u'\u276c' : u'\u276d', u'\u276e' : u'\u276f', u'\u2770' : u'\u2771', + u'\u2772' : u'\u2773', u'\u2774' : u'\u2775', u'\u27c3' : u'\u27c4', + u'\u27c5' : u'\u27c6', u'\u27d5' : u'\u27d6', u'\u27dd' : u'\u27de', + u'\u27e2' : u'\u27e3', u'\u27e4' : u'\u27e5', u'\u27e6' : u'\u27e7', + u'\u27e8' : u'\u27e9', u'\u27ea' : u'\u27eb', u'\u2983' : u'\u2984', + u'\u2985' : u'\u2986', u'\u2987' : u'\u2988', u'\u2989' : u'\u298a', + u'\u298b' : u'\u298c', u'\u298d' : u'\u298e', u'\u298f' : u'\u2990', + u'\u2991' : u'\u2992', u'\u2993' : u'\u2994', u'\u2995' : u'\u2996', + u'\u2997' : u'\u2998', u'\u29c0' : u'\u29c1', u'\u29c4' : u'\u29c5', + u'\u29cf' : u'\u29d0', u'\u29d1' : u'\u29d2', u'\u29d4' : u'\u29d5', + u'\u29d8' : u'\u29d9', u'\u29da' : u'\u29db', u'\u29f8' : u'\u29f9', + u'\u29fc' : u'\u29fd', u'\u2a2b' : u'\u2a2c', u'\u2a2d' : u'\u2a2e', + u'\u2a34' : u'\u2a35', u'\u2a3c' : u'\u2a3d', u'\u2a64' : u'\u2a65', + u'\u2a79' : u'\u2a7a', u'\u2a7d' : u'\u2a7e', u'\u2a7f' : u'\u2a80', + u'\u2a81' : u'\u2a82', u'\u2a83' : u'\u2a84', u'\u2a8b' : u'\u2a8c', + u'\u2a91' : u'\u2a92', u'\u2a93' : u'\u2a94', u'\u2a95' : u'\u2a96', + u'\u2a97' : u'\u2a98', u'\u2a99' : u'\u2a9a', u'\u2a9b' : u'\u2a9c', + u'\u2aa1' : u'\u2aa2', u'\u2aa6' : u'\u2aa7', u'\u2aa8' : u'\u2aa9', + u'\u2aaa' : u'\u2aab', u'\u2aac' : u'\u2aad', u'\u2aaf' : u'\u2ab0', + u'\u2ab3' : u'\u2ab4', u'\u2abb' : u'\u2abc', u'\u2abd' : u'\u2abe', + u'\u2abf' : u'\u2ac0', u'\u2ac1' : u'\u2ac2', u'\u2ac3' : u'\u2ac4', + u'\u2ac5' : u'\u2ac6', u'\u2acd' : u'\u2ace', u'\u2acf' : u'\u2ad0', + u'\u2ad1' : u'\u2ad2', u'\u2ad3' : u'\u2ad4', u'\u2ad5' : u'\u2ad6', + u'\u2aec' : u'\u2aed', u'\u2af7' : u'\u2af8', u'\u2af9' : u'\u2afa', + u'\u2e02' : u'\u2e03', u'\u2e04' : u'\u2e05', u'\u2e09' : u'\u2e0a', + u'\u2e0c' : u'\u2e0d', u'\u2e1c' : u'\u2e1d', u'\u2e20' : u'\u2e21', + u'\u3008' : u'\u3009', u'\u300a' : u'\u300b', u'\u300c' : u'\u300d', + u'\u300e' : u'\u300f', u'\u3010' : u'\u3011', u'\u3014' : u'\u3015', + u'\u3016' : u'\u3017', u'\u3018' : u'\u3019', u'\u301a' : u'\u301b', + u'\u301d' : u'\u301e', u'\ufd3e' : u'\ufd3f', u'\ufe17' : u'\ufe18', + u'\ufe35' : u'\ufe36', u'\ufe37' : u'\ufe38', u'\ufe39' : u'\ufe3a', + u'\ufe3b' : u'\ufe3c', u'\ufe3d' : u'\ufe3e', u'\ufe3f' : u'\ufe40', + u'\ufe41' : u'\ufe42', u'\ufe43' : u'\ufe44', u'\ufe47' : u'\ufe48', + u'\ufe59' : u'\ufe5a', u'\ufe5b' : u'\ufe5c', u'\ufe5d' : u'\ufe5e', + u'\uff08' : u'\uff09', u'\uff1c' : u'\uff1e', u'\uff3b' : u'\uff3d', + u'\uff5b' : u'\uff5d', u'\uff5f' : u'\uff60', u'\uff62' : u'\uff63', } def _build_word_match(words, boundary_regex_fragment = None, prefix = '', suffix = ''): if boundary_regex_fragment is None: - return r'\b(' + prefix + r'|'.join([ re.escape(x) for x in words]) + suffix + r')\b' + return r'\b(' + prefix + r'|'.join([ re.escape(x) for x in words]) + \ + suffix + r')\b' else: - return r'(?<!' + boundary_regex_fragment + ')' + prefix + '(' + \ - r'|'.join([ re.escape(x) for x in words]) + r')' + suffix + '(?!' + boundary_regex_fragment + ')' + return r'(?<!' + boundary_regex_fragment + r')' + prefix + r'(' + \ + r'|'.join([ re.escape(x) for x in words]) + r')' + suffix + r'(?!' + \ + boundary_regex_fragment + r')' def brackets_callback(token_class): def callback(lexer, match, context): @@ -2222,10 +2241,10 @@ class Perl6Lexer(ExtendedRegexLexer): context.pos = match.end() context.stack.append('root') - # If you're modifying these rules, be careful if you need to process '{' or '}' characters. - # We have special logic for processing these characters (due to the fact that you can nest - # Perl 6 code in regex blocks), so if you need to process one of them, make sure you also - # process the corresponding one! + # If you're modifying these rules, be careful if you need to process '{' or '}' + # characters. We have special logic for processing these characters (due to the fact + # that you can nest Perl 6 code in regex blocks), so if you need to process one of + # them, make sure you also process the corresponding one! tokens = { 'common' : [ (r'#[`|=](?P<delimiter>(?P<first_char>[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)', brackets_callback(Comment.Multiline)), @@ -2233,7 +2252,8 @@ class Perl6Lexer(ExtendedRegexLexer): (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline), (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline), (r'^=.*?\n\s*?\n', Comment.Multiline), - (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', bygroups(Keyword, Name), 'token-sym-brackets'), + (r'(regex|token|rule)(\s*' + PERL6_IDENTIFIER_RANGE + '+:sym)', + bygroups(Keyword, Name), 'token-sym-brackets'), (r'(regex|token|rule)(?!' + PERL6_IDENTIFIER_RANGE + ')(\s*' + PERL6_IDENTIFIER_RANGE + '+)?', bygroups(Keyword, Name), 'pre-token'), # deal with a special case in the Perl 6 grammar (role q { ... }) (r'(role)(\s+)(q)(\s*)', bygroups(Keyword, Text, Name, Text)), @@ -2241,24 +2261,28 @@ class Perl6Lexer(ExtendedRegexLexer): (_build_word_match(PERL6_BUILTIN_CLASSES, PERL6_IDENTIFIER_RANGE, suffix = '(?::[UD])?'), Name.Builtin), (_build_word_match(PERL6_BUILTINS, PERL6_IDENTIFIER_RANGE), Name.Builtin), # copied from PerlLexer - (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable), + (r'[$@%&][.^:?=!~]?' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', + Name.Variable), (r'\$[!/](?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global), (r'::\?\w+', Name.Variable.Global), - (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', Name.Variable.Global), + (r'[$@%&]\*' + PERL6_IDENTIFIER_RANGE + u'+(?:<<.*?>>|<.*?>|«.*?»)*', + Name.Variable.Global), (r'\$(?:<.*?>)+', Name.Variable), (r'(?:q|qq|Q)[a-zA-Z]?\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^0-9a-zA-Z:\s])(?P=first_char)*)', brackets_callback(String)), # copied from PerlLexer (r'0_?[0-7]+(_[0-7]+)*', Number.Oct), (r'0x[0-9A-Fa-f]+(_[0-9A-Fa-f]+)*', Number.Hex), (r'0b[01]+(_[01]+)*', Number.Bin), - (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?', Number.Float), + (r'(?i)(\d*(_\d*)*\.\d+(_\d*)*|\d+(_\d*)*\.\d+(_\d*)*)(e[+-]?\d+)?', + Number.Float), (r'(?i)\d+(_\d*)*e[+-]?\d+(_\d*)*', Number.Float), (r'\d+(_\d+)*', Number.Integer), (r'(?<=~~)\s*/(?:\\\\|\\/|.)*?/', String.Regex), (r'(?<=[=(,])\s*/(?:\\\\|\\/|.)*?/', String.Regex), (r'm\w+(?=\()', Name), (r'(?:m|ms|rx)\s*(?P<adverbs>:[\w\s:]+)?\s*(?P<delimiter>(?P<first_char>[^0-9a-zA-Z_:\s])(?P=first_char)*)', brackets_callback(String.Regex)), - (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/', String.Regex), + (r'(?:s|ss|tr)\s*(?::[\w\s:]+)?\s*/(?:\\\\|\\/|.)*?/(?:\\\\|\\/|.)*?/', + String.Regex), (r'<[^\s=].*?\S>', String), (_build_word_match(PERL6_OPERATORS), Operator), (r'[0-9a-zA-Z_]' + PERL6_IDENTIFIER_RANGE + '*', Name), @@ -2321,8 +2345,8 @@ class Perl6Lexer(ExtendedRegexLexer): rating = False # check for my/our/has declarations - # copied PERL6_IDENTIFIER_RANGE from above; not happy about that - if re.search("(?:my|our|has)\s+(?:['a-zA-Z0-9_:-]+\s+)?[$@%&(]", text): + if re.search("(?:my|our|has)\s+(?:" + Perl6Lexer.PERL6_IDENTIFIER_RANGE + \ + "+\s+)?[$@%&(]", text): rating = 0.8 saw_perl_decl = True @@ -2461,3 +2485,68 @@ class HyLexer(RegexLexer): def analyse_text(text): if '(import ' in text or '(defn ' in text: return 0.9 + + +class ChaiscriptLexer(RegexLexer): + """ + For `ChaiScript <http://chaiscript.com/>`_ source code. + + .. versionadded:: 2.0 + """ + + name = 'ChaiScript' + aliases = ['chai', 'chaiscript'] + filenames = ['*.chai'] + mimetypes = ['text/x-chaiscript', 'application/x-chaiscript'] + + flags = re.DOTALL + tokens = { + 'commentsandwhitespace': [ + (r'\s+', Text), + (r'//.*?\n', Comment.Single), + (r'/\*.*?\*/', Comment.Multiline), + (r'^\#.*?\n', Comment.Single) + ], + 'slashstartsregex': [ + include('commentsandwhitespace'), + (r'/(\\.|[^[/\\\n]|\[(\\.|[^\]\\\n])*])+/' + r'([gim]+\b|\B)', String.Regex, '#pop'), + (r'(?=/)', Text, ('#pop', 'badregex')), + (r'', Text, '#pop') + ], + 'badregex': [ + ('\n', Text, '#pop') + ], + 'root': [ + include('commentsandwhitespace'), + (r'\n', Text), + (r'[^\S\n]+', Text), + (r'\+\+|--|~|&&|\?|:|\|\||\\(?=\n)|\.\.' + r'(<<|>>>?|==?|!=?|[-<>+*%&\|\^/])=?', Operator, 'slashstartsregex'), + (r'[{(\[;,]', Punctuation, 'slashstartsregex'), + (r'[})\].]', Punctuation), + (r'[=+\-*/]', Operator), + (r'(for|in|while|do|break|return|continue|if|else|' + r'throw|try|catch' + r')\b', Keyword, 'slashstartsregex'), + (r'(var)\b', Keyword.Declaration, 'slashstartsregex'), + (r'(attr|def|fun)\b', Keyword.Reserved), + (r'(true|false)\b', Keyword.Constant), + (r'(eval|throw)\b', Name.Builtin), + (r'`\S+`', Name.Builtin), + (r'[$a-zA-Z_]\w*', Name.Other), + (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + (r'"', String.Double, 'dqstring'), + (r"'(\\\\|\\'|[^'])*'", String.Single), + ], + 'dqstring': [ + (r'\${[^"}]+?}', String.Iterpol), + (r'\$', String.Double), + (r'\\\\', String.Double), + (r'\\"', String.Double), + (r'[^\\\\\\"$]+', String.Double), + (r'"', String.Double, '#pop'), + ], + } diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index fc361e2f..35092f52 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -318,8 +318,8 @@ class NasmLexer(RegexLexer): filenames = ['*.asm', '*.ASM'] mimetypes = ['text/x-nasm'] - identifier = r'[a-zA-Z$._?][a-zA-Z0-9$._?#@~]*' - hexn = r'(?:0[xX][0-9a-fA-F]+|$0[0-9a-fA-F]*|[0-9]+[0-9a-fA-F]*h)' + identifier = r'[a-z$._?][\w$.?#@~]*' + hexn = r'(?:0[xX][0-9a-f]+|$0[0-9a-f]*|[0-9]+[0-9a-f]*h)' octn = r'[0-7]+q' binn = r'[01]+b' decn = r'[0-9]+' @@ -417,7 +417,7 @@ class Ca65Lexer(RegexLexer): r'|cl[cvdi]|se[cdi]|jmp|jsr|bne|beq|bpl|bmi|bvc|bvs|bcc|bcs' r'|p[lh][ap]|rt[is]|brk|nop|ta[xy]|t[xy]a|txs|tsx|and|ora|eor' r'|bit)\b', Keyword), - (r'\.[a-z0-9_]+', Keyword.Pseudo), + (r'\.\w+', Keyword.Pseudo), (r'[-+~*/^&|!<>=]', Operator), (r'"[^"\n]*.', String), (r"'[^'\n]*.", String.Char), diff --git a/pygments/lexers/compiled.py b/pygments/lexers/compiled.py index 947282dd..ec3015a5 100644 --- a/pygments/lexers/compiled.py +++ b/pygments/lexers/compiled.py @@ -56,8 +56,6 @@ class CFamilyLexer(RegexLexer): bygroups(using(this), Comment.Preproc), 'if0'), ('^(' + _ws1 + ')(#)', bygroups(using(this), Comment.Preproc), 'macro'), - (r'^(\s*)([a-zA-Z_][a-zA-Z0-9_]*)(\s*:)(?!:)', - bygroups(Text, Name.Label, Text)), (r'\n', Text), (r'\s+', Text), (r'\\\n', Text), # line continuation @@ -89,22 +87,23 @@ class CFamilyLexer(RegexLexer): r'declspec|finally|int64|try|leave|wchar_t|w64|unaligned|' r'raise|noop|identifier|forceinline|assume)\b', Keyword.Reserved), (r'(true|false|NULL)\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'([a-zA-Z_]\w*)(\s*)(:)(?!:)', bygroups(Name.Label, Text, Punctuation)), + ('[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), # functions - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')?({)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation), 'function'), # function declarations - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|[*]))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|[*]))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')?(;)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation)), @@ -224,7 +223,7 @@ class CppLexer(CFamilyLexer): (r'(__offload|__blockingoffload|__outer)\b', Keyword.Pseudo), ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], @@ -269,7 +268,7 @@ class PikeLexer(CppLexer): inherit, ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], @@ -290,9 +289,12 @@ class SwigLexer(CppLexer): tokens = { 'statements': [ - (r'(%[a-z_][a-z0-9_]*)', Name.Function), # SWIG directives - ('\$\**\&?[a-zA-Z0-9_]+', Name), # Special variables - (r'##*[a-zA-Z_][a-zA-Z0-9_]*', Comment.Preproc), # Stringification / additional preprocessor directives + # SWIG directives + (r'(%[a-z_][a-z0-9_]*)', Name.Function), + # Special variables + ('\$\**\&?\w+', Name), + # Stringification / additional preprocessor directives + (r'##*[a-zA-Z_]\w*', Comment.Preproc), inherit, ], } @@ -361,7 +363,7 @@ class ECLexer(CLexer): inherit, ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), # template specification (r'\s*(?=>)', Text, '#pop'), ], @@ -1156,7 +1158,7 @@ class DylanLexer(RegexLexer): 'type-error-value', 'type-for-copy', 'type-union', 'union', 'values', 'vector', 'zero?']) - valid_name = '\\\\?[a-zA-Z0-9' + re.escape('!&*<>|^$%@_-+~?/=') + ']+' + valid_name = '\\\\?[a-z0-9' + re.escape('!&*<>|^$%@_-+~?/=') + ']+' def get_tokens_unprocessed(self, text): for index, token, value in RegexLexer.get_tokens_unprocessed(self, text): @@ -1185,7 +1187,7 @@ class DylanLexer(RegexLexer): (r'//.*?\n', Comment.Single), # lid header - (r'([A-Za-z0-9-]+)(:)([ \t]*)(.*(?:\n[ \t].+)*)', + (r'([a-z0-9-]+)(:)([ \t]*)(.*(?:\n[ \t].+)*)', bygroups(Name.Attribute, Operator, Text, String)), ('', Text, 'code') # no header match, switch to code @@ -1202,7 +1204,7 @@ class DylanLexer(RegexLexer): # strings and characters (r'"', String, 'string'), - (r"'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), + (r"'(\\.|\\[0-7]{1,3}|\\x[a-f0-9]{1,2}|[^\\\'\n])'", String.Char), # binary integer (r'#[bB][01]+', Number), @@ -1217,7 +1219,7 @@ class DylanLexer(RegexLexer): (r'[-+]?\d+', Number.Integer), # hex integer - (r'#[xX][0-9a-fA-F]+', Number.Hex), + (r'#[xX][0-9a-f]+', Number.Hex), # Macro parameters (r'(\?' + valid_name + ')(:)' @@ -1241,7 +1243,7 @@ class DylanLexer(RegexLexer): (r'#"', String.Symbol, 'keyword'), # #rest, #key, #all-keys, etc. - (r'#[a-zA-Z0-9-]+', Keyword), + (r'#[a-z0-9-]+', Keyword), # required-init-keyword: style keywords. (valid_name + ':', Keyword), @@ -1270,7 +1272,7 @@ class DylanLexer(RegexLexer): ], 'string': [ (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', String.Escape), + (r'\\([\\abfnrtv"\']|x[a-f0-9]{2,4}|[0-7]{1,3})', String.Escape), (r'[^\\"\n]+', String), # all other characters (r'\\\n', String), # line continuation (r'\\', String), # stray backslash @@ -1372,9 +1374,9 @@ def objective(baselexer): # Matches [ <ws>? identifier <ws> ( identifier <ws>? ] | identifier? : ) # (note the identifier is *optional* when there is a ':'!) - _oc_message = re.compile(r'\[\s*[a-zA-Z_][a-zA-Z0-9_]*\s+' - r'(?:[a-zA-Z_][a-zA-Z0-9_]*\s*\]|' - r'(?:[a-zA-Z_][a-zA-Z0-9_]*)?:)') + _oc_message = re.compile(r'\[\s*[a-zA-Z_]\w*\s+' + r'(?:[a-zA-Z_]\w*\s*\]|' + r'(?:[a-zA-Z_]\w*)?:)') class GeneratedObjectiveCVariant(baselexer): """ @@ -1384,23 +1386,30 @@ def objective(baselexer): tokens = { 'statements': [ (r'@"', String, 'string'), + (r'@(YES|NO)', Number), (r"@'(\\.|\\[0-7]{1,3}|\\x[a-fA-F0-9]{1,2}|[^\\\'\n])'", String.Char), (r'@(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[lL]?', Number.Float), (r'@(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), (r'@0x[0-9a-fA-F]+[Ll]?', Number.Hex), (r'@0[0-7]+[Ll]?', Number.Oct), (r'@\d+[Ll]?', Number.Integer), - (r'@\([^()]+\)', Number), + (r'@\(', Literal, 'literal_number'), + (r'@\[', Literal, 'literal_array'), + (r'@\{', Literal, 'literal_dictionary'), (r'(@selector|@private|@protected|@public|@encode|' - r'@synchronized|@try|@throw|@catch|@finally|@end|@property|' + r'@synchronized|@try|@throw|@catch|@finally|@end|@property|@synthesize|' r'__bridge|__bridge_transfer|__autoreleasing|__block|__weak|__strong|' - r'weak|strong|retain|assign|unsafe_unretained|nonatomic|' - r'readonly|readwrite|setter|getter|typeof|in|out|inout|' - r'@synthesize|@dynamic|@optional|@required|@autoreleasepool)\b', Keyword), + r'weak|strong|copy|retain|assign|unsafe_unretained|atomic|nonatomic|' + r'readonly|readwrite|setter|getter|typeof|in|out|inout|release|class|' + r'@dynamic|@optional|@required|@autoreleasepool)\b', Keyword), (r'(id|instancetype|Class|IMP|SEL|BOOL|IBOutlet|IBAction|unichar)\b', Keyword.Type), (r'@(true|false|YES|NO)\n', Name.Builtin), (r'(YES|NO|nil|self|super)\b', Name.Builtin), + # Carbon types + (r'(Boolean|UInt8|SInt8|UInt16|SInt16|UInt32|SInt32)\b', Keyword.Type), + # Carbon built-ins + (r'(TRUE|FALSE)\b', Name.Builtin), (r'(@interface|@implementation)(\s+)', bygroups(Keyword, Text), ('#pop', 'oc_classname')), (r'(@class|@protocol)(\s+)', bygroups(Keyword, Text), @@ -1411,24 +1420,26 @@ def objective(baselexer): ], 'oc_classname' : [ # interface definition that inherits - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?(\s*)({)', - bygroups(Name.Class, Text, Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?', + ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?(\s*)({)', + bygroups(Name.Class, Text, Name.Class, Text, Punctuation), + ('#pop', 'oc_ivars')), + ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), # interface definition for a category - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\([a-zA-Z$_][a-zA-Z0-9$_]*\))(\s*)({)', - bygroups(Name.Class, Text, Name.Label, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\([a-zA-Z$_][a-zA-Z0-9$_]*\))', + ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))(\s*)({)', + bygroups(Name.Class, Text, Name.Label, Text, Punctuation), + ('#pop', 'oc_ivars')), + ('([a-zA-Z$_][\w$]*)(\s*)(\([a-zA-Z$_][\w$]*\))', bygroups(Name.Class, Text, Name.Label), '#pop'), # simple interface / implementation - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)({)', + ('([a-zA-Z$_][\w$]*)(\s*)({)', bygroups(Name.Class, Text, Punctuation), ('#pop', 'oc_ivars')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class, '#pop') + ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'oc_forward_classname' : [ - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*,\s*)', + ('([a-zA-Z$_][\w$]*)(\s*,\s*)', bygroups(Name.Class, Text), 'oc_forward_classname'), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*;?)', + ('([a-zA-Z$_][\w$]*)(\s*;?)', bygroups(Name.Class, Text), '#pop') ], 'oc_ivars' : [ @@ -1442,7 +1453,7 @@ def objective(baselexer): # methods (r'^([-+])(\s*)' # method marker r'(\(.*?\))?(\s*)' # return type - r'([a-zA-Z$_][a-zA-Z0-9$_]*:?)', # begin of method name + r'([a-zA-Z$_][\w$]*:?)', # begin of method name bygroups(Punctuation, Text, using(this), Text, Name.Function), 'method'), @@ -1454,13 +1465,37 @@ def objective(baselexer): # discussion in Issue 789 (r',', Punctuation), (r'\.\.\.', Punctuation), - (r'(\(.*?\))(\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)', + (r'(\(.*?\))(\s*)([a-zA-Z$_][\w$]*)', bygroups(using(this), Text, Name.Variable)), - (r'[a-zA-Z$_][a-zA-Z0-9$_]*:', Name.Function), + (r'[a-zA-Z$_][\w$]*:', Name.Function), (';', Punctuation, '#pop'), ('{', Punctuation, 'function'), ('', Text, '#pop'), ], + 'literal_number': [ + (r'\(', Punctuation, 'literal_number_inner'), + (r'\)', Literal, '#pop'), + include('statement'), + ], + 'literal_number_inner': [ + (r'\(', Punctuation, '#push'), + (r'\)', Punctuation, '#pop'), + include('statement'), + ], + 'literal_array': [ + (r'\[', Punctuation, 'literal_array_inner'), + (r'\]', Literal, '#pop'), + include('statement'), + ], + 'literal_array_inner': [ + (r'\[', Punctuation, '#push'), + (r'\]', Punctuation, '#pop'), + include('statement'), + ], + 'literal_dictionary': [ + (r'\}', Literal, '#pop'), + include('statement'), + ], } def analyse_text(text): @@ -1480,7 +1515,7 @@ def objective(baselexer): for index, token, value in \ baselexer.get_tokens_unprocessed(self, text): - if token is Name: + if token is Name or token is Name.Class: if value in COCOA_INTERFACES or value in COCOA_PROTOCOLS \ or value in COCOA_PRIMITIVES: token = Name.Builtin.Pseudo @@ -1538,7 +1573,7 @@ class FortranLexer(RegexLexer): (r'!.*\n', Comment), include('strings'), include('core'), - (r'[a-z][a-z0-9_]*', Name.Variable), + (r'[a-z]\w*', Name.Variable), include('nums'), (r'[\s]+', Text), ], @@ -1622,9 +1657,9 @@ class FortranLexer(RegexLexer): ], 'nums': [ - (r'\d+(?![.Ee])', Number.Integer), - (r'[+-]?\d*\.\d+([eE][-+]?\d+)?', Number.Float), - (r'[+-]?\d+\.\d*([eE][-+]?\d+)?', Number.Float), + (r'\d+(?![.e])(_[a-z]\w+)?', Number.Integer), + (r'[+-]?\d*\.\d+(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), + (r'[+-]?\d+\.\d*(e[-+]?\d+)?(_[a-z]\w+)?', Number.Float), ], } @@ -1713,20 +1748,20 @@ class PrologLexer(RegexLexer): (r'_', Keyword), # The don't-care variable (r'([a-z]+)(:)', bygroups(Name.Namespace, Punctuation)), (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[a-zA-Z0-9_$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(:-|-->)', bygroups(Name.Function, Text, Operator)), # function defn (u'([a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[a-zA-Z0-9_$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' + u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*)' u'(\\s*)(\\()', bygroups(Name.Function, Text, Punctuation)), (u'[a-z\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]' - u'[a-zA-Z0-9_$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', + u'[\w$\u00c0-\u1fff\u3040-\ud7ff\ue000-\uffef]*', String.Atom), # atom, characters # This one includes ! (u'[#&*+\\-./:<=>?@\\\\^~\u00a1-\u00bf\u2010-\u303f]+', String.Atom), # atom, graphics - (r'[A-Z_][A-Za-z0-9_]*', Name.Variable), + (r'[A-Z_]\w*', Name.Variable), (u'\\s+|[\u2000-\u200f\ufff0-\ufffe\uffef]', Text), ], 'nested-comment': [ @@ -1832,38 +1867,38 @@ class CythonLexer(RegexLexer): ('`.*?`', String.Backtick), ], 'name': [ - (r'@[a-zA-Z0-9_]+', Name.Decorator), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'@\w+', Name.Decorator), + ('[a-zA-Z_]\w*', Name), ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop') + ('[a-zA-Z_]\w*', Name.Function, '#pop') ], 'cdef': [ (r'(public|readonly|extern|api|inline)\b', Keyword.Reserved), (r'(struct|enum|union|class)\b', Keyword), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(?=[(:#=]|$)', + (r'([a-zA-Z_]\w*)(\s*)(?=[(:#=]|$)', bygroups(Name.Function, Text), '#pop'), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(,)', + (r'([a-zA-Z_]\w*)(\s*)(,)', bygroups(Name.Function, Text, Punctuation)), (r'from\b', Keyword, '#pop'), (r'as\b', Keyword), (r':', Punctuation, '#pop'), (r'(?=["\'])', Text, '#pop'), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Keyword.Type), + (r'[a-zA-Z_]\w*', Keyword.Type), (r'.', Text), ], 'classname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ (r'(\s+)(as)(\s+)', bygroups(Text, Keyword, Text)), - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_][\w.]*', Name.Namespace), (r'(\s*)(,)(\s*)', bygroups(Text, Operator, Text)), (r'', Text, '#pop') # all else: go back ], 'fromimport': [ (r'(\s+)(c?import)\b', bygroups(Text, Keyword), '#pop'), - (r'[a-zA-Z_.][a-zA-Z0-9_.]*', Name.Namespace), + (r'[a-zA-Z_.][\w.]*', Name.Namespace), # ``cdef foo from "header"``, or ``for foo from 0 < i < 10`` (r'', Text, '#pop'), ], @@ -1955,14 +1990,14 @@ class ValaLexer(RegexLexer): 'namespace'), (r'(class|errordomain|interface|struct)(\s+)', bygroups(Keyword.Declaration, Text), 'class'), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), # void is an actual keyword, others are in glib-2.0.vapi (r'(void|bool|char|double|float|int|int8|int16|int32|int64|long|' r'short|size_t|ssize_t|string|time_t|uchar|uint|uint8|uint16|' r'uint32|uint64|ulong|unichar|ushort)\b', Keyword.Type), (r'(true|false|null)\b', Name.Builtin), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), @@ -1988,10 +2023,10 @@ class ValaLexer(RegexLexer): (r'.*?\n', Comment), ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'namespace': [ - (r'[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace, '#pop') + (r'[a-zA-Z_][\w.]*', Name.Namespace, '#pop') ], } @@ -2015,9 +2050,9 @@ class OocLexer(RegexLexer): r'while|do|switch|case|as|in|version|return|true|false|null)\b', Keyword), (r'include\b', Keyword, 'include'), - (r'(cover)([ \t]+)(from)([ \t]+)([a-zA-Z0-9_]+[*@]?)', + (r'(cover)([ \t]+)(from)([ \t]+)(\w+[*@]?)', bygroups(Keyword, Text, Keyword, Text, Name.Class)), - (r'(func)((?:[ \t]|\\\n)+)(~[a-z_][a-zA-Z0-9_]*)', + (r'(func)((?:[ \t]|\\\n)+)(~[a-z_]\w*)', bygroups(Keyword, Text, Name.Function)), (r'\bfunc\b', Keyword), # Note: %= and ^= not listed on http://ooc-lang.org/syntax @@ -2028,11 +2063,11 @@ class OocLexer(RegexLexer): (r'(\.)([ \t]*)([a-z]\w*)', bygroups(Operator, Text, Name.Function)), (r'[A-Z][A-Z0-9_]+', Name.Constant), - (r'[A-Z][a-zA-Z0-9_]*([@*]|\[[ \t]*\])?', Name.Class), + (r'[A-Z]\w*([@*]|\[[ \t]*\])?', Name.Class), - (r'([a-z][a-zA-Z0-9_]*(?:~[a-z][a-zA-Z0-9_]*)?)((?:[ \t]|\\\n)*)(?=\()', + (r'([a-z]\w*(?:~[a-z]\w*)?)((?:[ \t]|\\\n)*)(?=\()', bygroups(Name.Function, Text)), - (r'[a-z][a-zA-Z0-9_]*', Name.Variable), + (r'[a-z]\w*', Name.Variable), # : introduces types (r'[:(){}\[\];,]', Punctuation), @@ -2414,7 +2449,7 @@ class AdaLexer(RegexLexer): (r'task|protected', Keyword.Declaration), (r'(subtype)(\s+)', bygroups(Keyword.Declaration, Text)), (r'(end)(\s+)', bygroups(Keyword.Reserved, Text), 'end'), - (r'(pragma)(\s+)([a-zA-Z0-9_]+)', bygroups(Keyword.Reserved, Text, + (r'(pragma)(\s+)(\w+)', bygroups(Keyword.Reserved, Text, Comment.Preproc)), (r'(true|false|null)\b', Keyword.Constant), (r'(Address|Byte|Boolean|Character|Controlled|Count|Cursor|' @@ -2457,7 +2492,7 @@ class AdaLexer(RegexLexer): (r'[0-9_]+', Number.Integer), ], 'attribute' : [ - (r"(')([a-zA-Z0-9_]+)", bygroups(Punctuation, Name.Attribute)), + (r"(')(\w+)", bygroups(Punctuation, Name.Attribute)), ], 'subprogram' : [ (r'\(', Punctuation, ('#pop', 'formal_part')), @@ -2468,7 +2503,7 @@ class AdaLexer(RegexLexer): ], 'end' : [ ('(if|case|record|loop|select)', Keyword.Reserved), - ('"[^"]+"|[a-zA-Z0-9_.]+', Name.Function), + ('"[^"]+"|[\w.]+', Name.Function), ('\s+', Text), (';', Punctuation, '#pop'), ], @@ -2508,7 +2543,7 @@ class AdaLexer(RegexLexer): ('is', Keyword.Reserved, '#pop'), (';', Punctuation, '#pop'), ('\(', Punctuation, 'package_instantiation'), - ('([a-zA-Z0-9_.]+)', Name.Class), + ('([\w.]+)', Name.Class), include('root'), ], 'package_instantiation': [ @@ -2551,7 +2586,7 @@ class Modula2Lexer(RegexLexer): (r'\s+', Text), # whitespace ], 'identifiers': [ - (r'([a-zA-Z_\$][a-zA-Z0-9_\$]*)', Name), + (r'([a-zA-Z_\$][\w\$]*)', Name), ], 'numliterals': [ (r'[01]+B', Number.Binary), # binary number (ObjM2) @@ -2729,9 +2764,9 @@ class BlitzMaxLexer(RegexLexer): bmax_vopwords = r'\b(Shl|Shr|Sar|Mod)\b' bmax_sktypes = r'@{1,2}|[!#$%]' bmax_lktypes = r'\b(Int|Byte|Short|Float|Double|Long)\b' - bmax_name = r'[a-z_][a-z0-9_]*' + bmax_name = r'[a-z_]\w*' bmax_var = (r'(%s)(?:(?:([ \t]*)(%s)|([ \t]*:[ \t]*\b(?:Shl|Shr|Sar|Mod)\b)' - r'|([ \t]*)([:])([ \t]*)(?:%s|(%s)))(?:([ \t]*)(Ptr))?)') % \ + r'|([ \t]*)(:)([ \t]*)(?:%s|(%s)))(?:([ \t]*)(Ptr))?)') % \ (bmax_name, bmax_sktypes, bmax_lktypes, bmax_name) bmax_func = bmax_var + r'?((?:[ \t]|\.\.\n)*)([(])' @@ -2824,7 +2859,7 @@ class BlitzBasicLexer(RegexLexer): r'Abs|Sgn|Handle|Int|Float|Str|' r'First|Last|Before|After)\b') bb_sktypes = r'@{1,2}|[#$%]' - bb_name = r'[a-z][a-z0-9_]*' + bb_name = r'[a-z]\w*' bb_var = (r'(%s)(?:([ \t]*)(%s)|([ \t]*)([.])([ \t]*)(?:(%s)))?') % \ (bb_name, bb_sktypes, bb_name) @@ -2969,7 +3004,7 @@ class NimrodLexer(RegexLexer): # Numbers (r'[0-9][0-9_]*(?=([eE.]|\'[fF](32|64)))', Number.Float, ('float-suffix', 'float-number')), - (r'0[xX][a-fA-F0-9][a-fA-F0-9_]*', Number.Hex, 'int-suffix'), + (r'0[xX][a-f0-9][a-f0-9_]*', Number.Hex, 'int-suffix'), (r'0[bB][01][01_]*', Number, 'int-suffix'), (r'0o[0-7][0-7_]*', Number.Oct, 'int-suffix'), (r'[0-9][0-9_]*', Number.Integer, 'int-suffix'), @@ -2978,7 +3013,7 @@ class NimrodLexer(RegexLexer): (r'.+$', Error), ], 'chars': [ - (r'\\([\\abcefnrtvl"\']|x[a-fA-F0-9]{2}|[0-9]{1,3})', String.Escape), + (r'\\([\\abcefnrtvl"\']|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape), (r"'", String.Char, '#pop'), (r".", String.Char) ], @@ -2992,7 +3027,7 @@ class NimrodLexer(RegexLexer): # newlines are an error (use "nl" state) ], 'dqs': [ - (r'\\([\\abcefnrtvl"\']|\n|x[a-fA-F0-9]{2}|[0-9]{1,3})', + (r'\\([\\abcefnrtvl"\']|\n|x[a-f0-9]{2}|[0-9]{1,3})', String.Escape), (r'"', String, '#pop'), include('strings') @@ -3048,7 +3083,7 @@ class FantomLexer(RegexLexer): dict ( pod = r'[\"\w\.]+', eos = r'\n|;', - id = r'[a-zA-Z_][a-zA-Z0-9_]*', + id = r'[a-zA-Z_]\w*', # all chars which can be part of type definition. Starts with # either letter, or [ (maps), or | (funcs) type = r'(?:\[|[a-zA-Z_]|\|)[:\w\[\]\|\->\?]*?', @@ -3333,7 +3368,7 @@ class RustLexer(RegexLexer): r"""|\\u[0-9a-fA-F]{4}|\\U[0-9a-fA-F]{8}|.)'""", String.Char), # Lifetime - (r"""'[a-zA-Z_][a-zA-Z0-9_]*""", Name.Label), + (r"""'[a-zA-Z_]\w*""", Name.Label), # Binary Literal (r'0b[01_]+', Number, 'number_lit'), # Octal Literal @@ -3461,10 +3496,10 @@ class MonkeyLexer(RegexLexer): filenames = ['*.monkey'] mimetypes = ['text/x-monkey'] - name_variable = r'[a-z_][a-zA-Z0-9_]*' - name_function = r'[A-Z][a-zA-Z0-9_]*' + name_variable = r'[a-z_]\w*' + name_function = r'[A-Z]\w*' name_constant = r'[A-Z_][A-Z0-9_]*' - name_class = r'[A-Z][a-zA-Z0-9_]*' + name_class = r'[A-Z]\w*' name_module = r'[a-z0-9_]*' keyword_type = r'(?:Int|Float|String|Bool|Object|Array|Void)' @@ -3806,12 +3841,12 @@ class LogosLexer(ObjectiveCppLexer): tokens = { 'statements': [ (r'(%orig|%log)\b', Keyword), - (r'(%c)\b(\()(\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(\))', + (r'(%c)\b(\()(\s*)([a-zA-Z$_][\w$]*)(\s*)(\))', bygroups(Keyword, Punctuation, Text, Name.Class, Text, Punctuation)), (r'(%init)\b(\()', bygroups(Keyword, Punctuation), 'logos_init_directive'), (r'(%init)(?=\s*;)', bygroups(Keyword)), - (r'(%hook|%group)(\s+)([a-zA-Z$_][a-zA-Z0-9$_]+)', + (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)', bygroups(Keyword, Text, Name.Class), '#pop'), (r'(%subclass)(\s+)', bygroups(Keyword, Text), ('#pop', 'logos_classname')), @@ -3820,20 +3855,20 @@ class LogosLexer(ObjectiveCppLexer): 'logos_init_directive' : [ ('\s+', Text), (',', Punctuation, ('logos_init_directive', '#pop')), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*)(=)(\s*)([^);]*)', + ('([a-zA-Z$_][\w$]*)(\s*)(=)(\s*)([^);]*)', bygroups(Name.Class, Text, Punctuation, Text, Text)), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class), + ('([a-zA-Z$_][\w$]*)', Name.Class), ('\)', Punctuation, '#pop'), ], 'logos_classname' : [ - ('([a-zA-Z$_][a-zA-Z0-9$_]*)(\s*:\s*)([a-zA-Z$_][a-zA-Z0-9$_]*)?', + ('([a-zA-Z$_][\w$]*)(\s*:\s*)([a-zA-Z$_][\w$]*)?', bygroups(Name.Class, Text, Name.Class), '#pop'), - ('([a-zA-Z$_][a-zA-Z0-9$_]*)', Name.Class, '#pop') + ('([a-zA-Z$_][\w$]*)', Name.Class, '#pop') ], 'root': [ (r'(%subclass)(\s+)', bygroups(Keyword, Text), 'logos_classname'), - (r'(%hook|%group)(\s+)([a-zA-Z$_][a-zA-Z0-9$_]+)', + (r'(%hook|%group)(\s+)([a-zA-Z$_][\w$]+)', bygroups(Keyword, Text, Name.Class)), (r'(%config)(\s*\(\s*)(\w+)(\s*=\s*)(.*?)(\s*\)\s*)', bygroups(Keyword, Text, Name.Variable, Text, String, Text)), @@ -3920,13 +3955,13 @@ class ChapelLexer(RegexLexer): (r'[:;,.?()\[\]{}]', Punctuation), # identifiers - (r'[a-zA-Z_][a-zA-Z0-9_$]*', Name.Other), + (r'[a-zA-Z_][\w$]*', Name.Other), ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_$]*', Name.Class, '#pop'), + (r'[a-zA-Z_][\w$]*', Name.Class, '#pop'), ], 'procname': [ - (r'[a-zA-Z_][a-zA-Z0-9_$]*', Name.Function, '#pop'), + (r'[a-zA-Z_][\w$]*', Name.Function, '#pop'), ], } @@ -3962,7 +3997,7 @@ class EiffelLexer(RegexLexer): (r"'([^'%]|%'|%%)'", String.Char), (r"(//|\\\\|>=|<=|:=|/=|~|/~|[\\\?!#%&@|+/\-=\>\*$<|^\[\]])", Operator), (r"([{}():;,.])", Punctuation), - (r'([a-z][a-zA-Z0-9_]*)|([A-Z][A-Z0-9_]*[a-z][a-zA-Z0-9_]*)', Name), + (r'([a-z]\w*)|([A-Z][A-Z0-9_]*[a-z]\w*)', Name), (r'([A-Z][A-Z0-9_]*)', Name.Class), (r'\n+', Text), ], diff --git a/pygments/lexers/dalvik.py b/pygments/lexers/dalvik.py index 901b7c5a..ea09cecb 100644 --- a/pygments/lexers/dalvik.py +++ b/pygments/lexers/dalvik.py @@ -9,6 +9,8 @@ :license: BSD, see LICENSE for details. """ +import re + from pygments.lexer import RegexLexer, include, bygroups from pygments.token import Keyword, Text, Comment, Name, String, Number, \ Punctuation @@ -73,22 +75,22 @@ class SmaliLexer(RegexLexer): (r'[0-9]+L?', Number.Integer), ], 'field': [ - (r'(\$?\b)([A-Za-z0-9_$]*)(:)', + (r'(\$?\b)([\w$]*)(:)', bygroups(Punctuation, Name.Variable, Punctuation)), ], 'method': [ (r'<(?:cl)?init>', Name.Function), # constructor - (r'(\$?\b)([A-Za-z0-9_$]*)(\()', + (r'(\$?\b)([\w$]*)(\()', bygroups(Punctuation, Name.Function, Punctuation)), ], 'label': [ - (r':[A-Za-z0-9_]+', Name.Label), + (r':\w+', Name.Label), ], 'class': [ # class names in the form Lcom/namespace/ClassName; # I only want to color the ClassName part, so the namespace part is # treated as 'Text' - (r'(L)((?:[A-Za-z0-9_$]+/)*)([A-Za-z0-9_$]+)(;)', + (r'(L)((?:[\w$]+/)*)([\w$]+)(;)', bygroups(Keyword.Type, Text, Name.Class, Text)), ], 'punctuation': [ @@ -102,3 +104,22 @@ class SmaliLexer(RegexLexer): (r'#.*?\n', Comment), ], } + + def analyse_text(text): + score = 0 + if re.search(r'^\s*\.class\s', text, re.MULTILINE): + score += 0.5 + if re.search(r'\b((check-cast|instance-of|throw-verification-error' + r')\b|(-to|add|[ais]get|[ais]put|and|cmpl|const|div|' + r'if|invoke|move|mul|neg|not|or|rem|return|rsub|shl|' + r'shr|sub|ushr)[-/])|{|}', text, re.MULTILINE): + score += 0.3 + if re.search(r'(\.(catchall|epilogue|restart local|prologue)|' + r'\b(array-data|class-change-error|declared-synchronized|' + r'(field|inline|vtable)@0x[0-9a-fA-F]|generic-error|' + r'illegal-class-access|illegal-field-access|' + r'illegal-method-access|instantiation-error|no-error|' + r'no-such-class|no-such-field|no-such-method|' + r'packed-switch|sparse-switch))\b', text, re.MULTILINE): + score += 0.6 + return score diff --git a/pygments/lexers/dotnet.py b/pygments/lexers/dotnet.py index 0754ba02..5a07cd0d 100644 --- a/pygments/lexers/dotnet.py +++ b/pygments/lexers/dotnet.py @@ -58,7 +58,7 @@ class CSharpLexer(RegexLexer): # see http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = { - 'none': '@?[_a-zA-Z][a-zA-Z0-9_]*', + 'none': '@?[_a-zA-Z]\w*', 'basic': ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' + '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'), @@ -170,7 +170,7 @@ class NemerleLexer(RegexLexer): # http://www.ecma-international.org/publications/files/ECMA-ST/Ecma-334.pdf levels = dict( - none = '@?[_a-zA-Z][a-zA-Z0-9_]*', + none = '@?[_a-zA-Z]\w*', basic = ('@?[_' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + ']' + '[' + uni.Lu + uni.Ll + uni.Lt + uni.Lm + uni.Nl + uni.Nd + uni.Pc + uni.Cf + uni.Mn + uni.Mc + ']*'), @@ -336,7 +336,7 @@ class BooLexer(RegexLexer): (r'"""(\\\\|\\"|.*?)"""', String.Double), (r'"(\\\\|\\"|[^"]*?)"', String.Double), (r"'(\\\\|\\'|[^']*?)'", String.Single), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*', Name), (r'(\d+\.\d*|\d*\.\d+)([fF][+-]?[0-9]+)?', Number.Float), (r'[0-9][0-9\.]*(ms?|d|h|s)', Number), (r'0\d+', Number.Oct), @@ -351,13 +351,13 @@ class BooLexer(RegexLexer): ('[*/]', Comment.Multiline) ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop') + ('[a-zA-Z_]\w*', Name.Function, '#pop') ], 'classname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'namespace': [ - ('[a-zA-Z_][a-zA-Z0-9_.]*', Name.Namespace, '#pop') + ('[a-zA-Z_][\w.]*', Name.Namespace, '#pop') ] } @@ -425,7 +425,7 @@ class VbNetLexer(RegexLexer): r'<=|>=|<>|[-&*/\\^+=<>]', Operator), ('"', String, 'string'), - ('[a-zA-Z_][a-zA-Z0-9_]*[%&@!#$]?', Name), + ('[a-z_]\w*[%&@!#$]?', Name), ('#.*?#', Literal.Date), (r'(\d+\.\d*|\d*\.\d+)([fF][+-]?[0-9]+)?', Number.Float), (r'\d+([SILDFR]|US|UI|UL)?', Number.Integer), @@ -439,17 +439,17 @@ class VbNetLexer(RegexLexer): (r'[^"]+', String), ], 'dim': [ - (r'[a-z_][a-z0-9_]*', Name.Variable, '#pop'), + (r'[a-z_]\w*', Name.Variable, '#pop'), (r'', Text, '#pop'), # any other syntax ], 'funcname': [ - (r'[a-z_][a-z0-9_]*', Name.Function, '#pop'), + (r'[a-z_]\w*', Name.Function, '#pop'), ], 'classname': [ - (r'[a-z_][a-z0-9_]*', Name.Class, '#pop'), + (r'[a-z_]\w*', Name.Class, '#pop'), ], 'namespace': [ - (r'[a-z_][a-z0-9_.]*', Name.Namespace, '#pop'), + (r'[a-z_][\w.]*', Name.Namespace, '#pop'), ], 'end': [ (r'\s+', Text), @@ -459,6 +459,12 @@ class VbNetLexer(RegexLexer): ] } + def analyse_text(text): + if (re.search(r'^[ \t]*#If', text, re.I) + or re.search(r'^[ \t]*(Module|Namespace)', re.I)): + return 0.5 + + class GenericAspxLexer(RegexLexer): """ @@ -589,9 +595,9 @@ class FSharpLexer(RegexLexer): 'root': [ (r'\s+', Text), (r'\(\)|\[\]', Name.Builtin.Pseudo), - (r'\b(?<!\.)([A-Z][A-Za-z0-9_\']*)(?=\s*\.)', + (r'\b(?<!\.)([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), - (r'\b([A-Z][A-Za-z0-9_\']*)', Name), + (r'\b([A-Z][\w\']*)', Name), (r'///.*?\n', String.Doc), (r'//.*?\n', Comment.Single), (r'\(\*(?!\))', Comment, 'comment'), @@ -600,13 +606,13 @@ class FSharpLexer(RegexLexer): (r'"""', String, 'tqs'), (r'"', String, 'string'), - (r'\b(open|module)(\s+)([a-zA-Z0-9_.]+)', + (r'\b(open|module)(\s+)([\w.]+)', bygroups(Keyword, Text, Name.Namespace)), - (r'\b(let!?)(\s+)([a-zA-Z0-9_]+)', + (r'\b(let!?)(\s+)(\w+)', bygroups(Keyword, Text, Name.Variable)), - (r'\b(type)(\s+)([a-zA-Z0-9_]+)', + (r'\b(type)(\s+)(\w+)', bygroups(Keyword, Text, Name.Class)), - (r'\b(member|override)(\s+)([a-zA-Z0-9_]+)(\.)([a-zA-Z0-9_]+)', + (r'\b(member|override)(\s+)(\w+)(\.)(\w+)', bygroups(Keyword, Text, Name, Punctuation, Name.Function)), (r'\b(%s)\b' % '|'.join(keywords), Keyword), (r'(%s)' % '|'.join(keyopts), Operator), @@ -635,9 +641,9 @@ class FSharpLexer(RegexLexer): 'dotted': [ (r'\s+', Text), (r'\.', Punctuation), - (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace), - (r'[A-Z][A-Za-z0-9_\']*', Name, '#pop'), - (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), # e.g. dictionary index access (r'', Text, '#pop'), ], diff --git a/pygments/lexers/functional.py b/pygments/lexers/functional.py index 122114fa..0c978254 100644 --- a/pygments/lexers/functional.py +++ b/pygments/lexers/functional.py @@ -14,12 +14,15 @@ import re from pygments.lexer import Lexer, RegexLexer, bygroups, include, do_insertions from pygments.token import Text, Comment, Operator, Keyword, Name, \ String, Number, Punctuation, Literal, Generic, Error +from pygments import unistring as uni -__all__ = ['RacketLexer', 'SchemeLexer', 'CommonLispLexer', 'HaskellLexer', - 'AgdaLexer', 'LiterateHaskellLexer', 'LiterateAgdaLexer', - 'SMLLexer', 'OcamlLexer', 'ErlangLexer', 'ErlangShellLexer', - 'OpaLexer', 'CoqLexer', 'NewLispLexer', 'NixLexer', 'ElixirLexer', - 'ElixirConsoleLexer', 'KokaLexer', 'IdrisLexer', 'LiterateIdrisLexer'] +__all__ = ['RacketLexer', 'SchemeLexer', 'CommonLispLexer', 'CryptolLexer', + 'HaskellLexer', 'AgdaLexer', 'LiterateCryptolLexer', + 'LiterateHaskellLexer', 'LiterateAgdaLexer', 'SMLLexer', + 'OcamlLexer', 'ErlangLexer', 'ErlangShellLexer', 'OpaLexer', + 'CoqLexer', 'NewLispLexer', 'NixLexer', 'ElixirLexer', + 'ElixirConsoleLexer', 'KokaLexer', 'IdrisLexer', + 'LiterateIdrisLexer'] line_re = re.compile('.*?\n') @@ -27,126 +30,219 @@ line_re = re.compile('.*?\n') class RacketLexer(RegexLexer): """ - Lexer for `Racket <http://racket-lang.org/>`_ source code (formerly known as - PLT Scheme). + Lexer for `Racket <http://racket-lang.org/>`_ source code (formerly + known as PLT Scheme). .. versionadded:: 1.6 """ name = 'Racket' aliases = ['racket', 'rkt'] - filenames = ['*.rkt', '*.rktl'] + filenames = ['*.rkt', '*.rktd', '*.rktl'] mimetypes = ['text/x-racket', 'application/x-racket'] - # From namespace-mapped-symbols - keywords = [ - '#%app', '#%datum', '#%expression', '#%module-begin', + # Generated by example.rkt + _keywords = [ + '#%app', '#%datum', '#%declare', '#%expression', '#%module-begin', '#%plain-app', '#%plain-lambda', '#%plain-module-begin', - '#%provide', '#%require', '#%stratified-body', '#%top', - '#%top-interaction', '#%variable-reference', '...', 'and', 'begin', - 'begin-for-syntax', 'begin0', 'case', 'case-lambda', 'cond', - 'datum->syntax-object', 'define', 'define-for-syntax', - 'define-struct', 'define-syntax', 'define-syntax-rule', - 'define-syntaxes', 'define-values', 'define-values-for-syntax', - 'delay', 'do', 'expand-path', 'fluid-let', 'hash-table-copy', - 'hash-table-count', 'hash-table-for-each', 'hash-table-get', - 'hash-table-iterate-first', 'hash-table-iterate-key', - 'hash-table-iterate-next', 'hash-table-iterate-value', - 'hash-table-map', 'hash-table-put!', 'hash-table-remove!', - 'hash-table?', 'if', 'lambda', 'let', 'let*', 'let*-values', - 'let-struct', 'let-syntax', 'let-syntaxes', 'let-values', 'let/cc', - 'let/ec', 'letrec', 'letrec-syntax', 'letrec-syntaxes', - 'letrec-syntaxes+values', 'letrec-values', 'list-immutable', - 'make-hash-table', 'make-immutable-hash-table', 'make-namespace', - 'module', 'module-identifier=?', 'module-label-identifier=?', - 'module-template-identifier=?', 'module-transformer-identifier=?', - 'namespace-transformer-require', 'or', 'parameterize', - 'parameterize*', 'parameterize-break', 'provide', - 'provide-for-label', 'provide-for-syntax', 'quasiquote', + '#%printing-module-begin', '#%provide', '#%require', + '#%stratified-body', '#%top', '#%top-interaction', + '#%variable-reference', '->', '->*', '->*m', '->d', '->dm', '->i', + '->m', '...', ':do-in', '==', '=>', '_', 'absent', 'abstract', + 'all-defined-out', 'all-from-out', 'and', 'any', 'augment', 'augment*', + 'augment-final', 'augment-final*', 'augride', 'augride*', 'begin', + 'begin-for-syntax', 'begin0', 'case', 'case->', 'case->m', + 'case-lambda', 'class', 'class*', 'class-field-accessor', + 'class-field-mutator', 'class/c', 'class/derived', 'combine-in', + 'combine-out', 'command-line', 'compound-unit', 'compound-unit/infer', + 'cond', 'contract', 'contract-out', 'contract-struct', 'contracted', + 'define', 'define-compound-unit', 'define-compound-unit/infer', + 'define-contract-struct', 'define-custom-hash-types', + 'define-custom-set-types', 'define-for-syntax', + 'define-local-member-name', 'define-logger', 'define-match-expander', + 'define-member-name', 'define-module-boundary-contract', + 'define-namespace-anchor', 'define-opt/c', 'define-sequence-syntax', + 'define-serializable-class', 'define-serializable-class*', + 'define-signature', 'define-signature-form', 'define-struct', + 'define-struct/contract', 'define-struct/derived', 'define-syntax', + 'define-syntax-rule', 'define-syntaxes', 'define-unit', + 'define-unit-binding', 'define-unit-from-context', + 'define-unit/contract', 'define-unit/new-import-export', + 'define-unit/s', 'define-values', 'define-values-for-export', + 'define-values-for-syntax', 'define-values/invoke-unit', + 'define-values/invoke-unit/infer', 'define/augment', + 'define/augment-final', 'define/augride', 'define/contract', + 'define/final-prop', 'define/match', 'define/overment', + 'define/override', 'define/override-final', 'define/private', + 'define/public', 'define/public-final', 'define/pubment', + 'define/subexpression-pos-prop', 'delay', 'delay/idle', 'delay/name', + 'delay/strict', 'delay/sync', 'delay/thread', 'do', 'else', 'except', + 'except-in', 'except-out', 'export', 'extends', 'failure-cont', + 'false', 'false/c', 'field', 'field-bound?', 'file', + 'flat-murec-contract', 'flat-rec-contract', 'for', 'for*', 'for*/and', + 'for*/first', 'for*/fold', 'for*/fold/derived', 'for*/hash', + 'for*/hasheq', 'for*/hasheqv', 'for*/last', 'for*/list', 'for*/lists', + 'for*/mutable-set', 'for*/mutable-seteq', 'for*/mutable-seteqv', + 'for*/or', 'for*/product', 'for*/set', 'for*/seteq', 'for*/seteqv', + 'for*/sum', 'for*/vector', 'for*/weak-set', 'for*/weak-seteq', + 'for*/weak-seteqv', 'for-label', 'for-meta', 'for-syntax', + 'for-template', 'for/and', 'for/first', 'for/fold', 'for/fold/derived', + 'for/hash', 'for/hasheq', 'for/hasheqv', 'for/last', 'for/list', + 'for/lists', 'for/mutable-set', 'for/mutable-seteq', + 'for/mutable-seteqv', 'for/or', 'for/product', 'for/set', 'for/seteq', + 'for/seteqv', 'for/sum', 'for/vector', 'for/weak-set', + 'for/weak-seteq', 'for/weak-seteqv', 'gen:custom-write', 'gen:dict', + 'gen:equal+hash', 'gen:set', 'gen:stream', 'generic', 'get-field', + 'if', 'implies', 'import', 'include', 'include-at/relative-to', + 'include-at/relative-to/reader', 'include/reader', 'inherit', + 'inherit-field', 'inherit/inner', 'inherit/super', 'init', + 'init-depend', 'init-field', 'init-rest', 'inner', 'inspect', + 'instantiate', 'interface', 'interface*', 'invoke-unit', + 'invoke-unit/infer', 'lambda', 'lazy', 'let', 'let*', 'let*-values', + 'let-syntax', 'let-syntaxes', 'let-values', 'let/cc', 'let/ec', + 'letrec', 'letrec-syntax', 'letrec-syntaxes', 'letrec-syntaxes+values', + 'letrec-values', 'lib', 'link', 'local', 'local-require', 'log-debug', + 'log-error', 'log-fatal', 'log-info', 'log-warning', 'match', 'match*', + 'match*/derived', 'match-define', 'match-define-values', + 'match-lambda', 'match-lambda*', 'match-lambda**', 'match-let', + 'match-let*', 'match-let*-values', 'match-let-values', 'match-letrec', + 'match/derived', 'match/values', 'member-name-key', 'method-contract?', + 'mixin', 'module', 'module*', 'module+', 'nand', 'new', 'nor', + 'object-contract', 'object/c', 'only', 'only-in', 'only-meta-in', + 'open', 'opt/c', 'or', 'overment', 'overment*', 'override', + 'override*', 'override-final', 'override-final*', 'parameterize', + 'parameterize*', 'parameterize-break', 'parametric->/c', 'place', + 'place*', 'planet', 'prefix', 'prefix-in', 'prefix-out', 'private', + 'private*', 'prompt-tag/c', 'protect-out', 'provide', + 'provide-signature-elements', 'provide/contract', 'public', 'public*', + 'public-final', 'public-final*', 'pubment', 'pubment*', 'quasiquote', 'quasisyntax', 'quasisyntax/loc', 'quote', 'quote-syntax', - 'quote-syntax/prune', 'require', 'require-for-label', - 'require-for-syntax', 'require-for-template', 'set!', - 'set!-values', 'syntax', 'syntax-case', 'syntax-case*', - 'syntax-id-rules', 'syntax-object->datum', 'syntax-rules', - 'syntax/loc', 'time', 'transcript-off', 'transcript-on', 'unless', - 'unquote', 'unquote-splicing', 'unsyntax', 'unsyntax-splicing', - 'when', 'with-continuation-mark', 'with-handlers', - 'with-handlers*', 'with-syntax', u'λ' + 'quote-syntax/prune', 'recontract-out', 'recursive-contract', + 'relative-in', 'rename', 'rename-in', 'rename-inner', 'rename-out', + 'rename-super', 'require', 'send', 'send*', 'send+', 'send-generic', + 'send/apply', 'send/keyword-apply', 'set!', 'set!-values', + 'set-field!', 'shared', 'stream', 'stream-cons', 'struct', 'struct*', + 'struct-copy', 'struct-field-index', 'struct-out', 'struct/c', + 'struct/ctc', 'struct/dc', 'submod', 'super', 'super-instantiate', + 'super-make-object', 'super-new', 'syntax', 'syntax-case', + 'syntax-case*', 'syntax-id-rules', 'syntax-rules', 'syntax/loc', 'tag', + 'this', 'this%', 'thunk', 'thunk*', 'time', 'unconstrained-domain->', + 'unit', 'unit-from-context', 'unit/c', 'unit/new-import-export', + 'unit/s', 'unless', 'unquote', 'unquote-splicing', 'unsyntax', + 'unsyntax-splicing', 'values/drop', 'when', 'with-continuation-mark', + 'with-contract', 'with-handlers', 'with-handlers*', 'with-method', + 'with-syntax', u'λ' ] - # From namespace-mapped-symbols - builtins = [ - '*', '+', '-', '/', '<', '<=', '=', '>', '>=', - 'abort-current-continuation', 'abs', 'absolute-path?', 'acos', - 'add1', 'alarm-evt', 'always-evt', 'andmap', 'angle', 'append', - 'apply', 'arithmetic-shift', 'arity-at-least', - 'arity-at-least-value', 'arity-at-least?', 'asin', 'assoc', 'assq', - 'assv', 'atan', 'banner', 'bitwise-and', 'bitwise-bit-field', - 'bitwise-bit-set?', 'bitwise-ior', 'bitwise-not', 'bitwise-xor', - 'boolean?', 'bound-identifier=?', 'box', 'box-immutable', 'box?', - 'break-enabled', 'break-thread', 'build-path', - 'build-path/convention-type', 'byte-pregexp', 'byte-pregexp?', + # Generated by example.rkt + _builtins = [ + '*', '+', '-', '/', '<', '</c', '<=', '<=/c', '=', '=/c', '>', '>/c', + '>=', '>=/c', 'abort-current-continuation', 'abs', 'absolute-path?', + 'acos', 'add-between', 'add1', 'alarm-evt', 'always-evt', 'and/c', + 'andmap', 'angle', 'any/c', 'append', 'append*', 'append-map', 'apply', + 'argmax', 'argmin', 'arithmetic-shift', 'arity-at-least', + 'arity-at-least-value', 'arity-at-least?', 'arity-checking-wrapper', + 'arity-includes?', 'arity=?', 'asin', 'assf', 'assoc', 'assq', 'assv', + 'atan', 'bad-number-of-results', 'banner', 'base->-doms/c', + 'base->-rngs/c', 'base->?', 'between/c', 'bitwise-and', + 'bitwise-bit-field', 'bitwise-bit-set?', 'bitwise-ior', 'bitwise-not', + 'bitwise-xor', 'blame-add-car-context', 'blame-add-cdr-context', + 'blame-add-context', 'blame-add-missing-party', + 'blame-add-nth-arg-context', 'blame-add-or-context', + 'blame-add-range-context', 'blame-add-unknown-context', + 'blame-context', 'blame-contract', 'blame-fmt->-string', + 'blame-negative', 'blame-original?', 'blame-positive', + 'blame-replace-negative', 'blame-source', 'blame-swap', + 'blame-swapped?', 'blame-update', 'blame-value', 'blame?', 'boolean=?', + 'boolean?', 'bound-identifier=?', 'box', 'box-cas!', 'box-immutable', + 'box-immutable/c', 'box/c', 'box?', 'break-enabled', 'break-thread', + 'build-chaperone-contract-property', 'build-compound-type-name', + 'build-contract-property', 'build-flat-contract-property', + 'build-list', 'build-path', 'build-path/convention-type', + 'build-string', 'build-vector', 'byte-pregexp', 'byte-pregexp?', 'byte-ready?', 'byte-regexp', 'byte-regexp?', 'byte?', 'bytes', 'bytes->immutable-bytes', 'bytes->list', 'bytes->path', - 'bytes->path-element', 'bytes->string/latin-1', - 'bytes->string/locale', 'bytes->string/utf-8', 'bytes-append', + 'bytes->path-element', 'bytes->string/latin-1', 'bytes->string/locale', + 'bytes->string/utf-8', 'bytes-append', 'bytes-append*', 'bytes-close-converter', 'bytes-convert', 'bytes-convert-end', - 'bytes-converter?', 'bytes-copy', 'bytes-copy!', 'bytes-fill!', - 'bytes-length', 'bytes-open-converter', 'bytes-ref', 'bytes-set!', - 'bytes-utf-8-index', 'bytes-utf-8-length', 'bytes-utf-8-ref', - 'bytes<?', 'bytes=?', 'bytes>?', 'bytes?', 'caaaar', 'caaadr', - 'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', 'cadadr', - 'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', - 'call-in-nested-thread', 'call-with-break-parameterization', - 'call-with-composable-continuation', - 'call-with-continuation-barrier', 'call-with-continuation-prompt', - 'call-with-current-continuation', 'call-with-escape-continuation', - 'call-with-exception-handler', - 'call-with-immediate-continuation-mark', 'call-with-input-file', - 'call-with-output-file', 'call-with-parameterization', - 'call-with-semaphore', 'call-with-semaphore/enable-break', - 'call-with-values', 'call/cc', 'call/ec', 'car', 'cdaaar', - 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', 'cdadr', 'cdar', 'cddaar', - 'cddadr', 'cddar', 'cdddar', 'cddddr', 'cdddr', 'cddr', 'cdr', - 'ceiling', 'channel-get', 'channel-put', 'channel-put-evt', - 'channel-try-get', 'channel?', 'chaperone-box', 'chaperone-evt', - 'chaperone-hash', 'chaperone-of?', 'chaperone-procedure', + 'bytes-converter?', 'bytes-copy', 'bytes-copy!', + 'bytes-environment-variable-name?', 'bytes-fill!', 'bytes-join', + 'bytes-length', 'bytes-no-nuls?', 'bytes-open-converter', 'bytes-ref', + 'bytes-set!', 'bytes-utf-8-index', 'bytes-utf-8-length', + 'bytes-utf-8-ref', 'bytes<?', 'bytes=?', 'bytes>?', 'bytes?', 'caaaar', + 'caaadr', 'caaar', 'caadar', 'caaddr', 'caadr', 'caar', 'cadaar', + 'cadadr', 'cadar', 'caddar', 'cadddr', 'caddr', 'cadr', + 'call-in-nested-thread', 'call-with-atomic-output-file', + 'call-with-break-parameterization', + 'call-with-composable-continuation', 'call-with-continuation-barrier', + 'call-with-continuation-prompt', 'call-with-current-continuation', + 'call-with-default-reading-parameterization', + 'call-with-escape-continuation', 'call-with-exception-handler', + 'call-with-file-lock/timeout', 'call-with-immediate-continuation-mark', + 'call-with-input-bytes', 'call-with-input-file', + 'call-with-input-file*', 'call-with-input-string', + 'call-with-output-bytes', 'call-with-output-file', + 'call-with-output-file*', 'call-with-output-string', + 'call-with-parameterization', 'call-with-semaphore', + 'call-with-semaphore/enable-break', 'call-with-values', 'call/cc', + 'call/ec', 'car', 'cdaaar', 'cdaadr', 'cdaar', 'cdadar', 'cdaddr', + 'cdadr', 'cdar', 'cddaar', 'cddadr', 'cddar', 'cdddar', 'cddddr', + 'cdddr', 'cddr', 'cdr', 'ceiling', 'channel-get', 'channel-put', + 'channel-put-evt', 'channel-put-evt?', 'channel-try-get', 'channel/c', + 'channel?', 'chaperone-box', 'chaperone-channel', + 'chaperone-continuation-mark-key', 'chaperone-contract-property?', + 'chaperone-contract?', 'chaperone-evt', 'chaperone-hash', + 'chaperone-of?', 'chaperone-procedure', 'chaperone-prompt-tag', 'chaperone-struct', 'chaperone-struct-type', 'chaperone-vector', 'chaperone?', 'char->integer', 'char-alphabetic?', 'char-blank?', 'char-ci<=?', 'char-ci<?', 'char-ci=?', 'char-ci>=?', 'char-ci>?', 'char-downcase', 'char-foldcase', 'char-general-category', 'char-graphic?', 'char-iso-control?', 'char-lower-case?', - 'char-numeric?', 'char-punctuation?', 'char-ready?', - 'char-symbolic?', 'char-title-case?', 'char-titlecase', - 'char-upcase', 'char-upper-case?', 'char-utf-8-length', - 'char-whitespace?', 'char<=?', 'char<?', 'char=?', 'char>=?', - 'char>?', 'char?', 'check-duplicate-identifier', - 'checked-procedure-check-and-extract', 'choice-evt', + 'char-numeric?', 'char-punctuation?', 'char-ready?', 'char-symbolic?', + 'char-title-case?', 'char-titlecase', 'char-upcase', + 'char-upper-case?', 'char-utf-8-length', 'char-whitespace?', 'char<=?', + 'char<?', 'char=?', 'char>=?', 'char>?', 'char?', + 'check-duplicate-identifier', 'checked-procedure-check-and-extract', + 'choice-evt', 'class->interface', 'class-info', 'class?', 'cleanse-path', 'close-input-port', 'close-output-port', - 'collect-garbage', 'collection-file-path', 'collection-path', - 'compile', 'compile-allow-set!-undefined', - 'compile-context-preservation-enabled', + 'coerce-chaperone-contract', 'coerce-chaperone-contracts', + 'coerce-contract', 'coerce-contract/f', 'coerce-contracts', + 'coerce-flat-contract', 'coerce-flat-contracts', 'collect-garbage', + 'collection-file-path', 'collection-path', 'compile', + 'compile-allow-set!-undefined', 'compile-context-preservation-enabled', 'compile-enforce-module-constants', 'compile-syntax', 'compiled-expression?', 'compiled-module-expression?', - 'complete-path?', 'complex?', 'cons', - 'continuation-mark-set->context', 'continuation-mark-set->list', - 'continuation-mark-set->list*', 'continuation-mark-set-first', - 'continuation-mark-set?', 'continuation-marks', - 'continuation-prompt-available?', 'continuation-prompt-tag?', - 'continuation?', 'copy-file', 'cos', - 'current-break-parameterization', 'current-code-inspector', - 'current-command-line-arguments', 'current-compile', - 'current-continuation-marks', 'current-custodian', - 'current-directory', 'current-drive', 'current-error-port', + 'complete-path?', 'complex?', 'compose', 'compose1', 'conjugate', + 'cons', 'cons/c', 'cons?', 'const', 'continuation-mark-key/c', + 'continuation-mark-key?', 'continuation-mark-set->context', + 'continuation-mark-set->list', 'continuation-mark-set->list*', + 'continuation-mark-set-first', 'continuation-mark-set?', + 'continuation-marks', 'continuation-prompt-available?', + 'continuation-prompt-tag?', 'continuation?', + 'contract-continuation-mark-key', 'contract-first-order', + 'contract-first-order-passes?', 'contract-name', 'contract-proc', + 'contract-projection', 'contract-property?', + 'contract-random-generate', 'contract-stronger?', + 'contract-struct-exercise', 'contract-struct-generate', + 'contract-val-first-projection', 'contract?', 'convert-stream', + 'copy-directory/files', 'copy-file', 'copy-port', 'cos', 'cosh', + 'count', 'current-blame-format', 'current-break-parameterization', + 'current-code-inspector', 'current-command-line-arguments', + 'current-compile', 'current-compiled-file-roots', + 'current-continuation-marks', 'current-contract-region', + 'current-custodian', 'current-directory', 'current-directory-for-user', + 'current-drive', 'current-environment-variables', 'current-error-port', 'current-eval', 'current-evt-pseudo-random-generator', - 'current-gc-milliseconds', 'current-get-interaction-input-port', - 'current-inexact-milliseconds', 'current-input-port', - 'current-inspector', 'current-library-collection-paths', + 'current-future', 'current-gc-milliseconds', + 'current-get-interaction-input-port', 'current-inexact-milliseconds', + 'current-input-port', 'current-inspector', + 'current-library-collection-links', 'current-library-collection-paths', 'current-load', 'current-load-extension', 'current-load-relative-directory', 'current-load/use-compiled', - 'current-locale', 'current-memory-use', 'current-milliseconds', - 'current-module-declare-name', 'current-module-declare-source', - 'current-module-name-resolver', 'current-namespace', + 'current-locale', 'current-logger', 'current-memory-use', + 'current-milliseconds', 'current-module-declare-name', + 'current-module-declare-source', 'current-module-name-resolver', + 'current-module-path-for-load', 'current-namespace', 'current-output-port', 'current-parameterization', 'current-preserved-thread-cell-values', 'current-print', 'current-process-milliseconds', 'current-prompt-read', @@ -155,133 +251,229 @@ class RacketLexer(RegexLexer): 'current-security-guard', 'current-subprocess-custodian-mode', 'current-thread', 'current-thread-group', 'current-thread-initial-stack-size', - 'current-write-relative-directory', 'custodian-box-value', - 'custodian-box?', 'custodian-limit-memory', + 'current-write-relative-directory', 'curry', 'curryr', + 'custodian-box-value', 'custodian-box?', 'custodian-limit-memory', 'custodian-managed-list', 'custodian-memory-accounting-available?', 'custodian-require-memory', 'custodian-shutdown-all', 'custodian?', 'custom-print-quotable-accessor', 'custom-print-quotable?', - 'custom-write-accessor', 'custom-write?', 'date', 'date*', - 'date*-nanosecond', 'date*-time-zone-name', 'date*?', 'date-day', - 'date-dst?', 'date-hour', 'date-minute', 'date-month', - 'date-second', 'date-time-zone-offset', 'date-week-day', - 'date-year', 'date-year-day', 'date?', 'datum-intern-literal', - 'default-continuation-prompt-tag', 'delete-directory', - 'delete-file', 'denominator', 'directory-exists?', - 'directory-list', 'display', 'displayln', 'dump-memory-stats', - 'dynamic-require', 'dynamic-require-for-syntax', 'dynamic-wind', - 'eof', 'eof-object?', 'ephemeron-value', 'ephemeron?', 'eprintf', - 'eq-hash-code', 'eq?', 'equal-hash-code', - 'equal-secondary-hash-code', 'equal?', 'equal?/recur', - 'eqv-hash-code', 'eqv?', 'error', 'error-display-handler', - 'error-escape-handler', 'error-print-context-length', - 'error-print-source-location', 'error-print-width', - 'error-value->string-handler', 'eval', 'eval-jit-enabled', - 'eval-syntax', 'even?', 'evt?', 'exact->inexact', 'exact-integer?', - 'exact-nonnegative-integer?', 'exact-positive-integer?', 'exact?', - 'executable-yield-handler', 'exit', 'exit-handler', 'exn', - 'exn-continuation-marks', 'exn-message', 'exn:break', - 'exn:break-continuation', 'exn:break?', 'exn:fail', - 'exn:fail:contract', 'exn:fail:contract:arity', - 'exn:fail:contract:arity?', 'exn:fail:contract:continuation', - 'exn:fail:contract:continuation?', - 'exn:fail:contract:divide-by-zero', + 'custom-write-accessor', 'custom-write-property-proc', 'custom-write?', + 'date', 'date*', 'date*-nanosecond', 'date*-time-zone-name', 'date*?', + 'date-day', 'date-dst?', 'date-hour', 'date-minute', 'date-month', + 'date-second', 'date-time-zone-offset', 'date-week-day', 'date-year', + 'date-year-day', 'date?', 'datum->syntax', 'datum-intern-literal', + 'default-continuation-prompt-tag', 'degrees->radians', + 'delete-directory', 'delete-directory/files', 'delete-file', + 'denominator', 'dict->list', 'dict-can-functional-set?', + 'dict-can-remove-keys?', 'dict-clear', 'dict-clear!', 'dict-copy', + 'dict-count', 'dict-empty?', 'dict-for-each', 'dict-has-key?', + 'dict-implements/c', 'dict-implements?', 'dict-iter-contract', + 'dict-iterate-first', 'dict-iterate-key', 'dict-iterate-next', + 'dict-iterate-value', 'dict-key-contract', 'dict-keys', 'dict-map', + 'dict-mutable?', 'dict-ref', 'dict-ref!', 'dict-remove', + 'dict-remove!', 'dict-set', 'dict-set!', 'dict-set*', 'dict-set*!', + 'dict-update', 'dict-update!', 'dict-value-contract', 'dict-values', + 'dict?', 'directory-exists?', 'directory-list', 'display', + 'display-lines', 'display-lines-to-file', 'display-to-file', + 'displayln', 'double-flonum?', 'drop', 'drop-right', 'dropf', + 'dropf-right', 'dump-memory-stats', 'dup-input-port', + 'dup-output-port', 'dynamic-get-field', 'dynamic-place', + 'dynamic-place*', 'dynamic-require', 'dynamic-require-for-syntax', + 'dynamic-send', 'dynamic-set-field!', 'dynamic-wind', 'eighth', + 'empty', 'empty-sequence', 'empty-stream', 'empty?', + 'environment-variables-copy', 'environment-variables-names', + 'environment-variables-ref', 'environment-variables-set!', + 'environment-variables?', 'eof', 'eof-evt', 'eof-object?', + 'ephemeron-value', 'ephemeron?', 'eprintf', 'eq-contract-val', + 'eq-contract?', 'eq-hash-code', 'eq?', 'equal-contract-val', + 'equal-contract?', 'equal-hash-code', 'equal-secondary-hash-code', + 'equal<%>', 'equal?', 'equal?/recur', 'eqv-hash-code', 'eqv?', 'error', + 'error-display-handler', 'error-escape-handler', + 'error-print-context-length', 'error-print-source-location', + 'error-print-width', 'error-value->string-handler', 'eval', + 'eval-jit-enabled', 'eval-syntax', 'even?', 'evt/c', 'evt?', + 'exact->inexact', 'exact-ceiling', 'exact-floor', 'exact-integer?', + 'exact-nonnegative-integer?', 'exact-positive-integer?', 'exact-round', + 'exact-truncate', 'exact?', 'executable-yield-handler', 'exit', + 'exit-handler', 'exn', 'exn-continuation-marks', 'exn-message', + 'exn:break', 'exn:break-continuation', 'exn:break:hang-up', + 'exn:break:hang-up?', 'exn:break:terminate', 'exn:break:terminate?', + 'exn:break?', 'exn:fail', 'exn:fail:contract', + 'exn:fail:contract:arity', 'exn:fail:contract:arity?', + 'exn:fail:contract:blame', 'exn:fail:contract:blame-object', + 'exn:fail:contract:blame?', 'exn:fail:contract:continuation', + 'exn:fail:contract:continuation?', 'exn:fail:contract:divide-by-zero', 'exn:fail:contract:divide-by-zero?', 'exn:fail:contract:non-fixnum-result', - 'exn:fail:contract:non-fixnum-result?', - 'exn:fail:contract:variable', 'exn:fail:contract:variable-id', - 'exn:fail:contract:variable?', 'exn:fail:contract?', - 'exn:fail:filesystem', 'exn:fail:filesystem:exists', - 'exn:fail:filesystem:exists?', 'exn:fail:filesystem:version', + 'exn:fail:contract:non-fixnum-result?', 'exn:fail:contract:variable', + 'exn:fail:contract:variable-id', 'exn:fail:contract:variable?', + 'exn:fail:contract?', 'exn:fail:filesystem', + 'exn:fail:filesystem:errno', 'exn:fail:filesystem:errno-errno', + 'exn:fail:filesystem:errno?', 'exn:fail:filesystem:exists', + 'exn:fail:filesystem:exists?', 'exn:fail:filesystem:missing-module', + 'exn:fail:filesystem:missing-module-path', + 'exn:fail:filesystem:missing-module?', 'exn:fail:filesystem:version', 'exn:fail:filesystem:version?', 'exn:fail:filesystem?', - 'exn:fail:network', 'exn:fail:network?', 'exn:fail:out-of-memory', - 'exn:fail:out-of-memory?', 'exn:fail:read', + 'exn:fail:network', 'exn:fail:network:errno', + 'exn:fail:network:errno-errno', 'exn:fail:network:errno?', + 'exn:fail:network?', 'exn:fail:object', 'exn:fail:object?', + 'exn:fail:out-of-memory', 'exn:fail:out-of-memory?', 'exn:fail:read', 'exn:fail:read-srclocs', 'exn:fail:read:eof', 'exn:fail:read:eof?', - 'exn:fail:read:non-char', 'exn:fail:read:non-char?', - 'exn:fail:read?', 'exn:fail:syntax', 'exn:fail:syntax-exprs', - 'exn:fail:syntax:unbound', 'exn:fail:syntax:unbound?', - 'exn:fail:syntax?', 'exn:fail:unsupported', + 'exn:fail:read:non-char', 'exn:fail:read:non-char?', 'exn:fail:read?', + 'exn:fail:syntax', 'exn:fail:syntax-exprs', + 'exn:fail:syntax:missing-module', + 'exn:fail:syntax:missing-module-path', + 'exn:fail:syntax:missing-module?', 'exn:fail:syntax:unbound', + 'exn:fail:syntax:unbound?', 'exn:fail:syntax?', 'exn:fail:unsupported', 'exn:fail:unsupported?', 'exn:fail:user', 'exn:fail:user?', - 'exn:fail?', 'exn:srclocs-accessor', 'exn:srclocs?', 'exn?', 'exp', - 'expand', 'expand-once', 'expand-syntax', 'expand-syntax-once', - 'expand-syntax-to-top-form', 'expand-to-top-form', - 'expand-user-path', 'expt', 'file-exists?', - 'file-or-directory-identity', 'file-or-directory-modify-seconds', - 'file-or-directory-permissions', 'file-position', 'file-size', - 'file-stream-buffer-mode', 'file-stream-port?', - 'filesystem-root-list', 'find-executable-path', - 'find-library-collection-paths', 'find-system-path', 'fixnum?', + 'exn:fail?', 'exn:misc:match?', 'exn:missing-module-accessor', + 'exn:missing-module?', 'exn:srclocs-accessor', 'exn:srclocs?', 'exn?', + 'exp', 'expand', 'expand-once', 'expand-syntax', 'expand-syntax-once', + 'expand-syntax-to-top-form', 'expand-to-top-form', 'expand-user-path', + 'explode-path', 'expt', 'externalizable<%>', 'false?', 'field-names', + 'fifth', 'file->bytes', 'file->bytes-lines', 'file->lines', + 'file->list', 'file->string', 'file->value', 'file-exists?', + 'file-name-from-path', 'file-or-directory-identity', + 'file-or-directory-modify-seconds', 'file-or-directory-permissions', + 'file-position', 'file-position*', 'file-size', + 'file-stream-buffer-mode', 'file-stream-port?', 'file-truncate', + 'filename-extension', 'filesystem-change-evt', + 'filesystem-change-evt-cancel', 'filesystem-change-evt?', + 'filesystem-root-list', 'filter', 'filter-map', 'filter-not', + 'filter-read-input-port', 'find-executable-path', 'find-files', + 'find-library-collection-links', 'find-library-collection-paths', + 'find-relative-path', 'find-system-path', 'findf', 'first', 'fixnum?', + 'flat-contract', 'flat-contract-predicate', 'flat-contract-property?', + 'flat-contract?', 'flat-named-contract', 'flatten', 'floating-point-bytes->real', 'flonum?', 'floor', 'flush-output', - 'for-each', 'force', 'format', 'fprintf', 'free-identifier=?', - 'gcd', 'generate-temporaries', 'gensym', 'get-output-bytes', - 'get-output-string', 'getenv', 'global-port-print-handler', - 'guard-evt', 'handle-evt', 'handle-evt?', 'hash', 'hash-equal?', - 'hash-eqv?', 'hash-has-key?', 'hash-placeholder?', 'hash-ref!', - 'hasheq', 'hasheqv', 'identifier-binding', - 'identifier-label-binding', 'identifier-prune-lexical-context', + 'fold-files', 'foldl', 'foldr', 'for-each', 'force', 'format', + 'fourth', 'fprintf', 'free-identifier=?', 'free-label-identifier=?', + 'free-template-identifier=?', 'free-transformer-identifier=?', + 'fsemaphore-count', 'fsemaphore-post', 'fsemaphore-try-wait?', + 'fsemaphore-wait', 'fsemaphore?', 'future', 'future?', + 'futures-enabled?', 'gcd', 'generate-member-key', + 'generate-temporaries', 'generic-set?', 'generic?', 'gensym', + 'get-output-bytes', 'get-output-string', 'get-preference', + 'get/build-val-first-projection', 'getenv', + 'global-port-print-handler', 'group-execute-bit', 'group-read-bit', + 'group-write-bit', 'guard-evt', 'handle-evt', 'handle-evt?', + 'has-contract?', 'hash', 'hash->list', 'hash-clear', 'hash-clear!', + 'hash-copy', 'hash-copy-clear', 'hash-count', 'hash-empty?', + 'hash-eq?', 'hash-equal?', 'hash-eqv?', 'hash-for-each', + 'hash-has-key?', 'hash-iterate-first', 'hash-iterate-key', + 'hash-iterate-next', 'hash-iterate-value', 'hash-keys', 'hash-map', + 'hash-placeholder?', 'hash-ref', 'hash-ref!', 'hash-remove', + 'hash-remove!', 'hash-set', 'hash-set!', 'hash-set*', 'hash-set*!', + 'hash-update', 'hash-update!', 'hash-values', 'hash-weak?', 'hash/c', + 'hash?', 'hasheq', 'hasheqv', 'identifier-binding', + 'identifier-binding-symbol', 'identifier-label-binding', + 'identifier-prune-lexical-context', 'identifier-prune-to-source-module', 'identifier-remove-from-definition-context', 'identifier-template-binding', 'identifier-transformer-binding', - 'identifier?', 'imag-part', 'immutable?', 'impersonate-box', - 'impersonate-hash', 'impersonate-procedure', 'impersonate-struct', - 'impersonate-vector', 'impersonator-of?', - 'impersonator-prop:application-mark', - 'impersonator-property-accessor-procedure?', - 'impersonator-property?', 'impersonator?', 'inexact->exact', - 'inexact-real?', 'inexact?', 'input-port?', 'inspector?', - 'integer->char', 'integer->integer-bytes', - 'integer-bytes->integer', 'integer-length', 'integer-sqrt', - 'integer-sqrt/remainder', 'integer?', + 'identifier?', 'identity', 'imag-part', 'immutable?', + 'impersonate-box', 'impersonate-channel', + 'impersonate-continuation-mark-key', 'impersonate-hash', + 'impersonate-procedure', 'impersonate-prompt-tag', + 'impersonate-struct', 'impersonate-vector', 'impersonator-contract?', + 'impersonator-ephemeron', 'impersonator-of?', + 'impersonator-prop:application-mark', 'impersonator-prop:contracted', + 'impersonator-property-accessor-procedure?', 'impersonator-property?', + 'impersonator?', 'implementation?', 'implementation?/c', 'in-bytes', + 'in-bytes-lines', 'in-cycle', 'in-dict', 'in-dict-keys', + 'in-dict-pairs', 'in-dict-values', 'in-directory', 'in-hash', + 'in-hash-keys', 'in-hash-pairs', 'in-hash-values', 'in-indexed', + 'in-input-port-bytes', 'in-input-port-chars', 'in-lines', 'in-list', + 'in-mlist', 'in-naturals', 'in-parallel', 'in-permutations', 'in-port', + 'in-producer', 'in-range', 'in-sequences', 'in-set', 'in-stream', + 'in-string', 'in-value', 'in-values*-sequence', 'in-values-sequence', + 'in-vector', 'inexact->exact', 'inexact-real?', 'inexact?', + 'infinite?', 'input-port-append', 'input-port?', 'inspector?', + 'instanceof/c', 'integer->char', 'integer->integer-bytes', + 'integer-bytes->integer', 'integer-in', 'integer-length', + 'integer-sqrt', 'integer-sqrt/remainder', 'integer?', + 'interface->method-names', 'interface-extension?', 'interface?', 'internal-definition-context-seal', 'internal-definition-context?', - 'keyword->string', 'keyword<?', 'keyword?', 'kill-thread', 'lcm', - 'length', 'liberal-define-context?', 'link-exists?', 'list', - 'list*', 'list->bytes', 'list->string', 'list->vector', 'list-ref', - 'list-tail', 'list?', 'load', 'load-extension', - 'load-on-demand-enabled', 'load-relative', - 'load-relative-extension', 'load/cd', 'load/use-compiled', - 'local-expand', 'local-expand/capture-lifts', - 'local-transformer-expand', - 'local-transformer-expand/capture-lifts', 'locale-string-encoding', - 'log', 'magnitude', 'make-arity-at-least', 'make-bytes', - 'make-channel', 'make-continuation-prompt-tag', 'make-custodian', - 'make-custodian-box', 'make-date', 'make-date*', - 'make-derived-parameter', 'make-directory', 'make-ephemeron', - 'make-exn', 'make-exn:break', 'make-exn:fail', - 'make-exn:fail:contract', 'make-exn:fail:contract:arity', + 'is-a?', 'is-a?/c', 'keyword->string', 'keyword-apply', 'keyword<?', + 'keyword?', 'keywords-match', 'kill-thread', 'last', 'last-pair', + 'lcm', 'length', 'liberal-define-context?', 'link-exists?', 'list', + 'list*', 'list->bytes', 'list->mutable-set', 'list->mutable-seteq', + 'list->mutable-seteqv', 'list->set', 'list->seteq', 'list->seteqv', + 'list->string', 'list->vector', 'list->weak-set', 'list->weak-seteq', + 'list->weak-seteqv', 'list-ref', 'list-tail', 'list/c', 'list?', + 'listof', 'load', 'load-extension', 'load-on-demand-enabled', + 'load-relative', 'load-relative-extension', 'load/cd', + 'load/use-compiled', 'local-expand', 'local-expand/capture-lifts', + 'local-transformer-expand', 'local-transformer-expand/capture-lifts', + 'locale-string-encoding', 'log', 'log-level?', 'log-max-level', + 'log-message', 'log-receiver?', 'logger-name', 'logger?', 'magnitude', + 'make-arity-at-least', 'make-base-empty-namespace', + 'make-base-namespace', 'make-bytes', 'make-channel', + 'make-chaperone-contract', 'make-continuation-mark-key', + 'make-continuation-prompt-tag', 'make-contract', 'make-custodian', + 'make-custodian-box', 'make-custom-hash', 'make-custom-hash-types', + 'make-custom-set', 'make-custom-set-types', 'make-date', 'make-date*', + 'make-derived-parameter', 'make-directory', 'make-directory*', + 'make-do-sequence', 'make-empty-namespace', + 'make-environment-variables', 'make-ephemeron', 'make-exn', + 'make-exn:break', 'make-exn:break:hang-up', 'make-exn:break:terminate', + 'make-exn:fail', 'make-exn:fail:contract', + 'make-exn:fail:contract:arity', 'make-exn:fail:contract:blame', 'make-exn:fail:contract:continuation', 'make-exn:fail:contract:divide-by-zero', 'make-exn:fail:contract:non-fixnum-result', 'make-exn:fail:contract:variable', 'make-exn:fail:filesystem', - 'make-exn:fail:filesystem:exists', + 'make-exn:fail:filesystem:errno', 'make-exn:fail:filesystem:exists', + 'make-exn:fail:filesystem:missing-module', 'make-exn:fail:filesystem:version', 'make-exn:fail:network', + 'make-exn:fail:network:errno', 'make-exn:fail:object', 'make-exn:fail:out-of-memory', 'make-exn:fail:read', 'make-exn:fail:read:eof', 'make-exn:fail:read:non-char', - 'make-exn:fail:syntax', 'make-exn:fail:syntax:unbound', - 'make-exn:fail:unsupported', 'make-exn:fail:user', - 'make-file-or-directory-link', 'make-hash-placeholder', - 'make-hasheq-placeholder', 'make-hasheqv', - 'make-hasheqv-placeholder', 'make-immutable-hasheqv', - 'make-impersonator-property', 'make-input-port', 'make-inspector', - 'make-known-char-range-list', 'make-output-port', 'make-parameter', - 'make-pipe', 'make-placeholder', 'make-polar', - 'make-prefab-struct', 'make-pseudo-random-generator', + 'make-exn:fail:syntax', 'make-exn:fail:syntax:missing-module', + 'make-exn:fail:syntax:unbound', 'make-exn:fail:unsupported', + 'make-exn:fail:user', 'make-file-or-directory-link', + 'make-flat-contract', 'make-fsemaphore', 'make-generic', + 'make-handle-get-preference-locked', 'make-hash', + 'make-hash-placeholder', 'make-hasheq', 'make-hasheq-placeholder', + 'make-hasheqv', 'make-hasheqv-placeholder', + 'make-immutable-custom-hash', 'make-immutable-hash', + 'make-immutable-hasheq', 'make-immutable-hasheqv', + 'make-impersonator-property', 'make-input-port', + 'make-input-port/read-to-peek', 'make-inspector', + 'make-keyword-procedure', 'make-known-char-range-list', + 'make-limited-input-port', 'make-list', 'make-lock-file-name', + 'make-log-receiver', 'make-logger', 'make-mixin-contract', + 'make-mutable-custom-set', 'make-none/c', 'make-object', + 'make-output-port', 'make-parameter', 'make-phantom-bytes', + 'make-pipe', 'make-pipe-with-specials', 'make-placeholder', + 'make-polar', 'make-prefab-struct', 'make-primitive-class', + 'make-proj-contract', 'make-pseudo-random-generator', 'make-reader-graph', 'make-readtable', 'make-rectangular', 'make-rename-transformer', 'make-resolved-module-path', 'make-security-guard', 'make-semaphore', 'make-set!-transformer', - 'make-shared-bytes', 'make-sibling-inspector', - 'make-special-comment', 'make-srcloc', 'make-string', - 'make-struct-field-accessor', 'make-struct-field-mutator', - 'make-struct-type', 'make-struct-type-property', - 'make-syntax-delta-introducer', 'make-syntax-introducer', - 'make-thread-cell', 'make-thread-group', 'make-vector', - 'make-weak-box', 'make-weak-hasheqv', 'make-will-executor', 'map', - 'max', 'mcar', 'mcdr', 'mcons', 'member', 'memq', 'memv', 'min', - 'module->exports', 'module->imports', 'module->language-info', - 'module->namespace', 'module-compiled-exports', + 'make-shared-bytes', 'make-sibling-inspector', 'make-special-comment', + 'make-srcloc', 'make-string', 'make-struct-field-accessor', + 'make-struct-field-mutator', 'make-struct-type', + 'make-struct-type-property', 'make-syntax-delta-introducer', + 'make-syntax-introducer', 'make-temporary-file', + 'make-tentative-pretty-print-output-port', 'make-thread-cell', + 'make-thread-group', 'make-vector', 'make-weak-box', + 'make-weak-custom-hash', 'make-weak-custom-set', 'make-weak-hash', + 'make-weak-hasheq', 'make-weak-hasheqv', 'make-will-executor', 'map', + 'match-equality-test', 'matches-arity-exactly?', 'max', 'mcar', 'mcdr', + 'mcons', 'member', 'member-name-key-hash-code', 'member-name-key=?', + 'member-name-key?', 'memf', 'memq', 'memv', 'merge-input', + 'method-in-interface?', 'min', 'mixin-contract', 'module->exports', + 'module->imports', 'module->language-info', 'module->namespace', + 'module-compiled-cross-phase-persistent?', 'module-compiled-exports', 'module-compiled-imports', 'module-compiled-language-info', - 'module-compiled-name', 'module-path-index-join', + 'module-compiled-name', 'module-compiled-submodules', + 'module-declared?', 'module-path-index-join', 'module-path-index-resolve', 'module-path-index-split', - 'module-path-index?', 'module-path?', 'module-predefined?', - 'module-provide-protected?', 'modulo', 'mpair?', 'nack-guard-evt', + 'module-path-index-submodule', 'module-path-index?', 'module-path?', + 'module-predefined?', 'module-provide-protected?', 'modulo', 'mpair?', + 'mutable-set', 'mutable-seteq', 'mutable-seteqv', 'n->th', + 'nack-guard-evt', 'namespace-anchor->empty-namespace', + 'namespace-anchor->namespace', 'namespace-anchor?', 'namespace-attach-module', 'namespace-attach-module-declaration', 'namespace-base-phase', 'namespace-mapped-symbols', 'namespace-module-identifier', 'namespace-module-registry', @@ -289,136 +481,212 @@ class RacketLexer(RegexLexer): 'namespace-require/copy', 'namespace-require/expansion-time', 'namespace-set-variable-value!', 'namespace-symbol->identifier', 'namespace-syntax-introduce', 'namespace-undefine-variable!', - 'namespace-unprotect-module', 'namespace-variable-value', - 'namespace?', 'negative?', 'never-evt', 'newline', - 'normal-case-path', 'not', 'null', 'null?', 'number->string', - 'number?', 'numerator', 'object-name', 'odd?', 'open-input-bytes', + 'namespace-unprotect-module', 'namespace-variable-value', 'namespace?', + 'nan?', 'natural-number/c', 'negate', 'negative?', 'never-evt', + u'new-∀/c', u'new-∃/c', 'newline', 'ninth', 'non-empty-listof', + 'none/c', 'normal-case-path', 'normalize-arity', 'normalize-path', + 'normalized-arity?', 'not', 'not/c', 'null', 'null?', 'number->string', + 'number?', 'numerator', 'object%', 'object->vector', 'object-info', + 'object-interface', 'object-method-arity-includes?', 'object-name', + 'object=?', 'object?', 'odd?', 'one-of/c', 'open-input-bytes', 'open-input-file', 'open-input-output-file', 'open-input-string', - 'open-output-bytes', 'open-output-file', 'open-output-string', - 'ormap', 'output-port?', 'pair?', 'parameter-procedure=?', - 'parameter?', 'parameterization?', 'path->bytes', - 'path->complete-path', 'path->directory-path', 'path->string', - 'path-add-suffix', 'path-convention-type', 'path-element->bytes', - 'path-element->string', 'path-for-some-system?', - 'path-list-string->path-list', 'path-replace-suffix', - 'path-string?', 'path?', 'peek-byte', 'peek-byte-or-special', - 'peek-bytes', 'peek-bytes!', 'peek-bytes-avail!', - 'peek-bytes-avail!*', 'peek-bytes-avail!/enable-break', - 'peek-char', 'peek-char-or-special', 'peek-string', 'peek-string!', - 'pipe-content-length', 'placeholder-get', 'placeholder-set!', - 'placeholder?', 'poll-guard-evt', 'port-closed-evt', - 'port-closed?', 'port-commit-peeked', 'port-count-lines!', - 'port-count-lines-enabled', 'port-display-handler', - 'port-file-identity', 'port-file-unlock', 'port-next-location', - 'port-print-handler', 'port-progress-evt', - 'port-provides-progress-evts?', 'port-read-handler', - 'port-try-file-lock?', 'port-write-handler', 'port-writes-atomic?', - 'port-writes-special?', 'port?', 'positive?', - 'prefab-key->struct-type', 'prefab-struct-key', 'pregexp', - 'pregexp?', 'primitive-closure?', 'primitive-result-arity', - 'primitive?', 'print', 'print-as-expression', + 'open-output-bytes', 'open-output-file', 'open-output-nowhere', + 'open-output-string', 'or/c', 'order-of-magnitude', 'ormap', + 'other-execute-bit', 'other-read-bit', 'other-write-bit', + 'output-port?', 'pair?', 'parameter-procedure=?', 'parameter/c', + 'parameter?', 'parameterization?', 'parse-command-line', 'partition', + 'path->bytes', 'path->complete-path', 'path->directory-path', + 'path->string', 'path-add-suffix', 'path-convention-type', + 'path-element->bytes', 'path-element->string', 'path-element?', + 'path-for-some-system?', 'path-list-string->path-list', 'path-only', + 'path-replace-suffix', 'path-string?', 'path<?', 'path?', + 'pathlist-closure', 'peek-byte', 'peek-byte-or-special', 'peek-bytes', + 'peek-bytes!', 'peek-bytes!-evt', 'peek-bytes-avail!', + 'peek-bytes-avail!*', 'peek-bytes-avail!-evt', + 'peek-bytes-avail!/enable-break', 'peek-bytes-evt', 'peek-char', + 'peek-char-or-special', 'peek-string', 'peek-string!', + 'peek-string!-evt', 'peek-string-evt', 'peeking-input-port', + 'permutations', 'phantom-bytes?', 'pi', 'pi.f', 'pipe-content-length', + 'place-break', 'place-channel', 'place-channel-get', + 'place-channel-put', 'place-channel-put/get', 'place-channel?', + 'place-dead-evt', 'place-enabled?', 'place-kill', 'place-location?', + 'place-message-allowed?', 'place-sleep', 'place-wait', 'place?', + 'placeholder-get', 'placeholder-set!', 'placeholder?', + 'poll-guard-evt', 'port->bytes', 'port->bytes-lines', 'port->lines', + 'port->list', 'port->string', 'port-closed-evt', 'port-closed?', + 'port-commit-peeked', 'port-count-lines!', 'port-count-lines-enabled', + 'port-counts-lines?', 'port-display-handler', 'port-file-identity', + 'port-file-unlock', 'port-next-location', 'port-print-handler', + 'port-progress-evt', 'port-provides-progress-evts?', + 'port-read-handler', 'port-try-file-lock?', 'port-write-handler', + 'port-writes-atomic?', 'port-writes-special?', 'port?', 'positive?', + 'predicate/c', 'prefab-key->struct-type', 'prefab-key?', + 'prefab-struct-key', 'preferences-lock-file-mode', 'pregexp', + 'pregexp?', 'pretty-display', 'pretty-format', 'pretty-print', + 'pretty-print-.-symbol-without-bars', + 'pretty-print-abbreviate-read-macros', 'pretty-print-columns', + 'pretty-print-current-style-table', 'pretty-print-depth', + 'pretty-print-exact-as-decimal', 'pretty-print-extend-style-table', + 'pretty-print-handler', 'pretty-print-newline', + 'pretty-print-post-print-hook', 'pretty-print-pre-print-hook', + 'pretty-print-print-hook', 'pretty-print-print-line', + 'pretty-print-remap-stylable', 'pretty-print-show-inexactness', + 'pretty-print-size-hook', 'pretty-print-style-table?', + 'pretty-printing', 'pretty-write', 'primitive-closure?', + 'primitive-result-arity', 'primitive?', 'print', 'print-as-expression', 'print-boolean-long-form', 'print-box', 'print-graph', 'print-hash-table', 'print-mpair-curly-braces', 'print-pair-curly-braces', 'print-reader-abbreviations', 'print-struct', 'print-syntax-width', 'print-unreadable', - 'print-vector-length', 'printf', 'procedure->method', - 'procedure-arity', 'procedure-arity-includes?', 'procedure-arity?', + 'print-vector-length', 'printable/c', 'printable<%>', 'printf', + 'procedure->method', 'procedure-arity', 'procedure-arity-includes/c', + 'procedure-arity-includes?', 'procedure-arity?', 'procedure-closure-contents-eq?', 'procedure-extract-target', - 'procedure-reduce-arity', 'procedure-rename', - 'procedure-struct-type?', 'procedure?', 'promise?', - 'prop:arity-string', 'prop:checked-procedure', - 'prop:custom-print-quotable', 'prop:custom-write', - 'prop:equal+hash', 'prop:evt', 'prop:exn:srclocs', + 'procedure-keywords', 'procedure-reduce-arity', + 'procedure-reduce-keyword-arity', 'procedure-rename', + 'procedure-struct-type?', 'procedure?', 'process', 'process*', + 'process*/ports', 'process/ports', 'processor-count', 'progress-evt?', + 'promise-forced?', 'promise-running?', 'promise/c', 'promise?', + 'prop:arity-string', 'prop:chaperone-contract', + 'prop:checked-procedure', 'prop:contract', 'prop:contracted', + 'prop:custom-print-quotable', 'prop:custom-write', 'prop:dict', + 'prop:dict/contract', 'prop:equal+hash', 'prop:evt', + 'prop:exn:missing-module', 'prop:exn:srclocs', 'prop:flat-contract', 'prop:impersonator-of', 'prop:input-port', - 'prop:liberal-define-context', 'prop:output-port', - 'prop:procedure', 'prop:rename-transformer', - 'prop:set!-transformer', 'pseudo-random-generator->vector', + 'prop:liberal-define-context', 'prop:opt-chaperone-contract', + 'prop:opt-chaperone-contract-get-test', 'prop:opt-chaperone-contract?', + 'prop:output-port', 'prop:place-location', 'prop:procedure', + 'prop:rename-transformer', 'prop:sequence', 'prop:set!-transformer', + 'prop:stream', 'proper-subset?', 'pseudo-random-generator->vector', 'pseudo-random-generator-vector?', 'pseudo-random-generator?', - 'putenv', 'quotient', 'quotient/remainder', 'raise', - 'raise-arity-error', 'raise-mismatch-error', 'raise-syntax-error', - 'raise-type-error', 'raise-user-error', 'random', 'random-seed', - 'rational?', 'rationalize', 'read', 'read-accept-bar-quote', - 'read-accept-box', 'read-accept-compiled', 'read-accept-dot', - 'read-accept-graph', 'read-accept-infix-dot', 'read-accept-lang', - 'read-accept-quasiquote', 'read-accept-reader', 'read-byte', - 'read-byte-or-special', 'read-bytes', 'read-bytes!', - 'read-bytes-avail!', 'read-bytes-avail!*', - 'read-bytes-avail!/enable-break', 'read-bytes-line', - 'read-case-sensitive', 'read-char', 'read-char-or-special', - 'read-curly-brace-as-paren', 'read-decimal-as-inexact', - 'read-eval-print-loop', 'read-language', 'read-line', - 'read-on-demand-source', 'read-square-bracket-as-paren', - 'read-string', 'read-string!', 'read-syntax', + 'put-preferences', 'putenv', 'quotient', 'quotient/remainder', + 'radians->degrees', 'raise', 'raise-argument-error', + 'raise-arguments-error', 'raise-arity-error', 'raise-blame-error', + 'raise-contract-error', 'raise-mismatch-error', + 'raise-not-cons-blame-error', 'raise-range-error', + 'raise-result-error', 'raise-syntax-error', 'raise-type-error', + 'raise-user-error', 'random', 'random-seed', 'range', 'rational?', + 'rationalize', 'read', 'read-accept-bar-quote', 'read-accept-box', + 'read-accept-compiled', 'read-accept-dot', 'read-accept-graph', + 'read-accept-infix-dot', 'read-accept-lang', 'read-accept-quasiquote', + 'read-accept-reader', 'read-byte', 'read-byte-or-special', + 'read-bytes', 'read-bytes!', 'read-bytes!-evt', 'read-bytes-avail!', + 'read-bytes-avail!*', 'read-bytes-avail!-evt', + 'read-bytes-avail!/enable-break', 'read-bytes-evt', 'read-bytes-line', + 'read-bytes-line-evt', 'read-case-sensitive', 'read-char', + 'read-char-or-special', 'read-curly-brace-as-paren', + 'read-decimal-as-inexact', 'read-eval-print-loop', 'read-language', + 'read-line', 'read-line-evt', 'read-on-demand-source', + 'read-square-bracket-as-paren', 'read-string', 'read-string!', + 'read-string!-evt', 'read-string-evt', 'read-syntax', 'read-syntax/recursive', 'read/recursive', 'readtable-mapping', - 'readtable?', 'real->double-flonum', 'real->floating-point-bytes', - 'real->single-flonum', 'real-part', 'real?', 'regexp', - 'regexp-match', 'regexp-match-peek', 'regexp-match-peek-immediate', - 'regexp-match-peek-positions', + 'readtable?', 'real->decimal-string', 'real->double-flonum', + 'real->floating-point-bytes', 'real->single-flonum', 'real-in', + 'real-part', 'real?', 'reencode-input-port', 'reencode-output-port', + 'regexp', 'regexp-match', 'regexp-match*', 'regexp-match-evt', + 'regexp-match-exact?', 'regexp-match-peek', + 'regexp-match-peek-immediate', 'regexp-match-peek-positions', + 'regexp-match-peek-positions*', 'regexp-match-peek-positions-immediate', 'regexp-match-peek-positions-immediate/end', 'regexp-match-peek-positions/end', 'regexp-match-positions', - 'regexp-match-positions/end', 'regexp-match/end', 'regexp-match?', - 'regexp-max-lookbehind', 'regexp-replace', 'regexp-replace*', - 'regexp?', 'relative-path?', 'remainder', + 'regexp-match-positions*', 'regexp-match-positions/end', + 'regexp-match/end', 'regexp-match?', 'regexp-max-lookbehind', + 'regexp-quote', 'regexp-replace', 'regexp-replace*', + 'regexp-replace-quote', 'regexp-replaces', 'regexp-split', + 'regexp-try-match', 'regexp?', 'relative-path?', 'relocate-input-port', + 'relocate-output-port', 'remainder', 'remove', 'remove*', + 'remove-duplicates', 'remq', 'remq*', 'remv', 'remv*', 'rename-file-or-directory', 'rename-transformer-target', - 'rename-transformer?', 'resolve-path', 'resolved-module-path-name', - 'resolved-module-path?', 'reverse', 'round', 'seconds->date', - 'security-guard?', 'semaphore-peek-evt', 'semaphore-post', - 'semaphore-try-wait?', 'semaphore-wait', - 'semaphore-wait/enable-break', 'semaphore?', - 'set!-transformer-procedure', 'set!-transformer?', 'set-box!', - 'set-mcar!', 'set-mcdr!', 'set-port-next-location!', - 'shared-bytes', 'shell-execute', 'simplify-path', 'sin', - 'single-flonum?', 'sleep', 'special-comment-value', - 'special-comment?', 'split-path', 'sqrt', 'srcloc', - 'srcloc-column', 'srcloc-line', 'srcloc-position', 'srcloc-source', - 'srcloc-span', 'srcloc?', 'string', 'string->bytes/latin-1', - 'string->bytes/locale', 'string->bytes/utf-8', + 'rename-transformer?', 'reroot-path', 'resolve-path', + 'resolved-module-path-name', 'resolved-module-path?', 'rest', + 'reverse', 'round', 'second', 'seconds->date', 'security-guard?', + 'semaphore-peek-evt', 'semaphore-peek-evt?', 'semaphore-post', + 'semaphore-try-wait?', 'semaphore-wait', 'semaphore-wait/enable-break', + 'semaphore?', 'sequence->list', 'sequence->stream', + 'sequence-add-between', 'sequence-andmap', 'sequence-append', + 'sequence-count', 'sequence-filter', 'sequence-fold', + 'sequence-for-each', 'sequence-generate', 'sequence-generate*', + 'sequence-length', 'sequence-map', 'sequence-ormap', 'sequence-ref', + 'sequence-tail', 'sequence?', 'set', 'set!-transformer-procedure', + 'set!-transformer?', 'set->list', 'set->stream', 'set-add', 'set-add!', + 'set-box!', 'set-clear', 'set-clear!', 'set-copy', 'set-copy-clear', + 'set-count', 'set-empty?', 'set-eq?', 'set-equal?', 'set-eqv?', + 'set-first', 'set-for-each', 'set-implements/c', 'set-implements?', + 'set-intersect', 'set-intersect!', 'set-map', 'set-mcar!', 'set-mcdr!', + 'set-member?', 'set-mutable?', 'set-phantom-bytes!', + 'set-port-next-location!', 'set-remove', 'set-remove!', 'set-rest', + 'set-subtract', 'set-subtract!', 'set-symmetric-difference', + 'set-symmetric-difference!', 'set-union', 'set-union!', 'set-weak?', + 'set/c', 'set=?', 'set?', 'seteq', 'seteqv', 'seventh', 'sgn', + 'shared-bytes', 'shell-execute', 'shrink-path-wrt', 'shuffle', + 'simple-form-path', 'simplify-path', 'sin', 'single-flonum?', 'sinh', + 'sixth', 'skip-projection-wrapper?', 'sleep', + 'some-system-path->string', 'sort', 'special-comment-value', + 'special-comment?', 'special-filter-input-port', 'split-at', + 'split-at-right', 'split-path', 'splitf-at', 'splitf-at-right', 'sqr', + 'sqrt', 'srcloc', 'srcloc->string', 'srcloc-column', 'srcloc-line', + 'srcloc-position', 'srcloc-source', 'srcloc-span', 'srcloc?', + 'stop-after', 'stop-before', 'stream->list', 'stream-add-between', + 'stream-andmap', 'stream-append', 'stream-count', 'stream-empty?', + 'stream-filter', 'stream-first', 'stream-fold', 'stream-for-each', + 'stream-length', 'stream-map', 'stream-ormap', 'stream-ref', + 'stream-rest', 'stream-tail', 'stream?', 'string', + 'string->bytes/latin-1', 'string->bytes/locale', 'string->bytes/utf-8', 'string->immutable-string', 'string->keyword', 'string->list', 'string->number', 'string->path', 'string->path-element', - 'string->symbol', 'string->uninterned-symbol', - 'string->unreadable-symbol', 'string-append', 'string-ci<=?', - 'string-ci<?', 'string-ci=?', 'string-ci>=?', 'string-ci>?', - 'string-copy', 'string-copy!', 'string-downcase', 'string-fill!', - 'string-foldcase', 'string-length', 'string-locale-ci<?', - 'string-locale-ci=?', 'string-locale-ci>?', - 'string-locale-downcase', 'string-locale-upcase', + 'string->some-system-path', 'string->symbol', + 'string->uninterned-symbol', 'string->unreadable-symbol', + 'string-append', 'string-append*', 'string-ci<=?', 'string-ci<?', + 'string-ci=?', 'string-ci>=?', 'string-ci>?', 'string-copy', + 'string-copy!', 'string-downcase', 'string-environment-variable-name?', + 'string-fill!', 'string-foldcase', 'string-join', 'string-len/c', + 'string-length', 'string-locale-ci<?', 'string-locale-ci=?', + 'string-locale-ci>?', 'string-locale-downcase', 'string-locale-upcase', 'string-locale<?', 'string-locale=?', 'string-locale>?', - 'string-normalize-nfc', 'string-normalize-nfd', - 'string-normalize-nfkc', 'string-normalize-nfkd', 'string-ref', - 'string-set!', 'string-titlecase', 'string-upcase', - 'string-utf-8-length', 'string<=?', 'string<?', 'string=?', - 'string>=?', 'string>?', 'string?', 'struct->vector', + 'string-no-nuls?', 'string-normalize-nfc', 'string-normalize-nfd', + 'string-normalize-nfkc', 'string-normalize-nfkd', + 'string-normalize-spaces', 'string-ref', 'string-replace', + 'string-set!', 'string-split', 'string-titlecase', 'string-trim', + 'string-upcase', 'string-utf-8-length', 'string<=?', 'string<?', + 'string=?', 'string>=?', 'string>?', 'string?', 'struct->vector', 'struct-accessor-procedure?', 'struct-constructor-procedure?', 'struct-info', 'struct-mutator-procedure?', 'struct-predicate-procedure?', 'struct-type-info', 'struct-type-make-constructor', 'struct-type-make-predicate', - 'struct-type-property-accessor-procedure?', + 'struct-type-property-accessor-procedure?', 'struct-type-property/c', 'struct-type-property?', 'struct-type?', 'struct:arity-at-least', 'struct:date', 'struct:date*', 'struct:exn', 'struct:exn:break', + 'struct:exn:break:hang-up', 'struct:exn:break:terminate', 'struct:exn:fail', 'struct:exn:fail:contract', - 'struct:exn:fail:contract:arity', + 'struct:exn:fail:contract:arity', 'struct:exn:fail:contract:blame', 'struct:exn:fail:contract:continuation', 'struct:exn:fail:contract:divide-by-zero', 'struct:exn:fail:contract:non-fixnum-result', 'struct:exn:fail:contract:variable', 'struct:exn:fail:filesystem', + 'struct:exn:fail:filesystem:errno', 'struct:exn:fail:filesystem:exists', + 'struct:exn:fail:filesystem:missing-module', 'struct:exn:fail:filesystem:version', 'struct:exn:fail:network', + 'struct:exn:fail:network:errno', 'struct:exn:fail:object', 'struct:exn:fail:out-of-memory', 'struct:exn:fail:read', 'struct:exn:fail:read:eof', 'struct:exn:fail:read:non-char', - 'struct:exn:fail:syntax', 'struct:exn:fail:syntax:unbound', - 'struct:exn:fail:unsupported', 'struct:exn:fail:user', - 'struct:srcloc', 'struct?', 'sub1', 'subbytes', 'subprocess', - 'subprocess-group-enabled', 'subprocess-kill', 'subprocess-pid', - 'subprocess-status', 'subprocess-wait', 'subprocess?', 'substring', - 'symbol->string', 'symbol-interned?', 'symbol-unreadable?', - 'symbol?', 'sync', 'sync/enable-break', 'sync/timeout', - 'sync/timeout/enable-break', 'syntax->list', 'syntax-arm', - 'syntax-column', 'syntax-disarm', 'syntax-e', 'syntax-line', - 'syntax-local-bind-syntaxes', 'syntax-local-certifier', - 'syntax-local-context', 'syntax-local-expand-expression', - 'syntax-local-get-shadower', 'syntax-local-introduce', - 'syntax-local-lift-context', 'syntax-local-lift-expression', + 'struct:exn:fail:syntax', 'struct:exn:fail:syntax:missing-module', + 'struct:exn:fail:syntax:unbound', 'struct:exn:fail:unsupported', + 'struct:exn:fail:user', 'struct:srcloc', + 'struct:wrapped-extra-arg-arrow', 'struct?', 'sub1', 'subbytes', + 'subclass?', 'subclass?/c', 'subprocess', 'subprocess-group-enabled', + 'subprocess-kill', 'subprocess-pid', 'subprocess-status', + 'subprocess-wait', 'subprocess?', 'subset?', 'substring', + 'symbol->string', 'symbol-interned?', 'symbol-unreadable?', 'symbol<?', + 'symbol=?', 'symbol?', 'symbols', 'sync', 'sync/enable-break', + 'sync/timeout', 'sync/timeout/enable-break', 'syntax->datum', + 'syntax->list', 'syntax-arm', 'syntax-column', 'syntax-disarm', + 'syntax-e', 'syntax-line', 'syntax-local-bind-syntaxes', + 'syntax-local-certifier', 'syntax-local-context', + 'syntax-local-expand-expression', 'syntax-local-get-shadower', + 'syntax-local-introduce', 'syntax-local-lift-context', + 'syntax-local-lift-expression', 'syntax-local-lift-module-end-declaration', 'syntax-local-lift-provide', 'syntax-local-lift-require', 'syntax-local-lift-values-expression', @@ -427,163 +695,229 @@ class RacketLexer(RegexLexer): 'syntax-local-module-defined-identifiers', 'syntax-local-module-exports', 'syntax-local-module-required-identifiers', 'syntax-local-name', - 'syntax-local-phase-level', + 'syntax-local-phase-level', 'syntax-local-submodules', 'syntax-local-transforming-module-provides?', 'syntax-local-value', - 'syntax-local-value/immediate', 'syntax-original?', - 'syntax-position', 'syntax-property', - 'syntax-property-symbol-keys', 'syntax-protect', 'syntax-rearm', - 'syntax-recertify', 'syntax-shift-phase-level', 'syntax-source', - 'syntax-source-module', 'syntax-span', 'syntax-taint', + 'syntax-local-value/immediate', 'syntax-original?', 'syntax-position', + 'syntax-property', 'syntax-property-symbol-keys', 'syntax-protect', + 'syntax-rearm', 'syntax-recertify', 'syntax-shift-phase-level', + 'syntax-source', 'syntax-source-module', 'syntax-span', 'syntax-taint', 'syntax-tainted?', 'syntax-track-origin', 'syntax-transforming-module-expression?', 'syntax-transforming?', - 'syntax?', 'system-big-endian?', 'system-idle-evt', - 'system-language+country', 'system-library-subpath', - 'system-path-convention-type', 'system-type', 'tan', - 'tcp-abandon-port', 'tcp-accept', 'tcp-accept-evt', - 'tcp-accept-ready?', 'tcp-accept/enable-break', 'tcp-addresses', - 'tcp-close', 'tcp-connect', 'tcp-connect/enable-break', - 'tcp-listen', 'tcp-listener?', 'tcp-port?', 'terminal-port?', - 'thread', 'thread-cell-ref', 'thread-cell-set!', 'thread-cell?', - 'thread-dead-evt', 'thread-dead?', 'thread-group?', - 'thread-resume', 'thread-resume-evt', 'thread-rewind-receive', - 'thread-running?', 'thread-suspend', 'thread-suspend-evt', + 'syntax/c', 'syntax?', 'system', 'system*', 'system*/exit-code', + 'system-big-endian?', 'system-idle-evt', 'system-language+country', + 'system-library-subpath', 'system-path-convention-type', 'system-type', + 'system/exit-code', 'tail-marks-match?', 'take', 'take-right', 'takef', + 'takef-right', 'tan', 'tanh', 'tcp-abandon-port', 'tcp-accept', + 'tcp-accept-evt', 'tcp-accept-ready?', 'tcp-accept/enable-break', + 'tcp-addresses', 'tcp-close', 'tcp-connect', + 'tcp-connect/enable-break', 'tcp-listen', 'tcp-listener?', 'tcp-port?', + 'tentative-pretty-print-port-cancel', + 'tentative-pretty-print-port-transfer', 'tenth', 'terminal-port?', + 'the-unsupplied-arg', 'third', 'thread', 'thread-cell-ref', + 'thread-cell-set!', 'thread-cell-values?', 'thread-cell?', + 'thread-dead-evt', 'thread-dead?', 'thread-group?', 'thread-receive', + 'thread-receive-evt', 'thread-resume', 'thread-resume-evt', + 'thread-rewind-receive', 'thread-running?', 'thread-send', + 'thread-suspend', 'thread-suspend-evt', 'thread-try-receive', 'thread-wait', 'thread/suspend-to-kill', 'thread?', 'time-apply', - 'truncate', 'udp-addresses', 'udp-bind!', 'udp-bound?', - 'udp-close', 'udp-connect!', 'udp-connected?', 'udp-open-socket', - 'udp-receive!', 'udp-receive!*', 'udp-receive!-evt', - 'udp-receive!/enable-break', 'udp-receive-ready-evt', 'udp-send', - 'udp-send*', 'udp-send-evt', 'udp-send-ready-evt', 'udp-send-to', - 'udp-send-to*', 'udp-send-to-evt', 'udp-send-to/enable-break', - 'udp-send/enable-break', 'udp?', 'unbox', - 'uncaught-exception-handler', 'use-collection-link-paths', + 'touch', 'transplant-input-port', 'transplant-output-port', 'true', + 'truncate', 'udp-addresses', 'udp-bind!', 'udp-bound?', 'udp-close', + 'udp-connect!', 'udp-connected?', 'udp-multicast-interface', + 'udp-multicast-join-group!', 'udp-multicast-leave-group!', + 'udp-multicast-loopback?', 'udp-multicast-set-interface!', + 'udp-multicast-set-loopback!', 'udp-multicast-set-ttl!', + 'udp-multicast-ttl', 'udp-open-socket', 'udp-receive!', + 'udp-receive!*', 'udp-receive!-evt', 'udp-receive!/enable-break', + 'udp-receive-ready-evt', 'udp-send', 'udp-send*', 'udp-send-evt', + 'udp-send-ready-evt', 'udp-send-to', 'udp-send-to*', 'udp-send-to-evt', + 'udp-send-to/enable-break', 'udp-send/enable-break', 'udp?', 'unbox', + 'uncaught-exception-handler', 'unit?', 'unspecified-dom', + 'unsupplied-arg?', 'use-collection-link-paths', 'use-compiled-file-paths', 'use-user-specific-search-paths', - 'values', 'variable-reference->empty-namespace', + 'user-execute-bit', 'user-read-bit', 'user-write-bit', + 'value-contract', 'values', 'variable-reference->empty-namespace', 'variable-reference->module-base-phase', 'variable-reference->module-declaration-inspector', - 'variable-reference->module-source', - 'variable-reference->namespace', 'variable-reference->phase', + 'variable-reference->module-path-index', + 'variable-reference->module-source', 'variable-reference->namespace', + 'variable-reference->phase', 'variable-reference->resolved-module-path', 'variable-reference-constant?', 'variable-reference?', 'vector', 'vector->immutable-vector', 'vector->list', - 'vector->pseudo-random-generator', - 'vector->pseudo-random-generator!', 'vector->values', - 'vector-fill!', 'vector-immutable', 'vector-length', 'vector-ref', - 'vector-set!', 'vector-set-performance-stats!', 'vector?', - 'version', 'void', 'void?', 'weak-box-value', 'weak-box?', - 'will-execute', 'will-executor?', 'will-register', - 'will-try-execute', 'with-input-from-file', 'with-output-to-file', - 'wrap-evt', 'write', 'write-byte', 'write-bytes', + 'vector->pseudo-random-generator', 'vector->pseudo-random-generator!', + 'vector->values', 'vector-append', 'vector-argmax', 'vector-argmin', + 'vector-copy', 'vector-copy!', 'vector-count', 'vector-drop', + 'vector-drop-right', 'vector-fill!', 'vector-filter', + 'vector-filter-not', 'vector-immutable', 'vector-immutable/c', + 'vector-immutableof', 'vector-length', 'vector-map', 'vector-map!', + 'vector-member', 'vector-memq', 'vector-memv', 'vector-ref', + 'vector-set!', 'vector-set*!', 'vector-set-performance-stats!', + 'vector-split-at', 'vector-split-at-right', 'vector-take', + 'vector-take-right', 'vector/c', 'vector?', 'vectorof', 'version', + 'void', 'void?', 'weak-box-value', 'weak-box?', 'weak-set', + 'weak-seteq', 'weak-seteqv', 'will-execute', 'will-executor?', + 'will-register', 'will-try-execute', 'with-input-from-bytes', + 'with-input-from-file', 'with-input-from-string', + 'with-output-to-bytes', 'with-output-to-file', 'with-output-to-string', + 'would-be-future', 'wrap-evt', 'wrapped-extra-arg-arrow', + 'wrapped-extra-arg-arrow-extra-neg-party-argument', + 'wrapped-extra-arg-arrow-real-func', 'wrapped-extra-arg-arrow?', + 'writable<%>', 'write', 'write-byte', 'write-bytes', 'write-bytes-avail', 'write-bytes-avail*', 'write-bytes-avail-evt', 'write-bytes-avail/enable-break', 'write-char', 'write-special', - 'write-special-avail*', 'write-special-evt', 'write-string', 'zero?' + 'write-special-avail*', 'write-special-evt', 'write-string', + 'write-to-file', 'xor', 'zero?', '~.a', '~.s', '~.v', '~a', '~e', '~r', + '~s', '~v' ] - # From SchemeLexer - valid_name = r'[a-zA-Z0-9!$%&*+,/:<=>?@^_~|-]+' + _opening_parenthesis = r'[([{]' + _closing_parenthesis = r'[)\]}]' + _delimiters = r'()[\]{}",\'`;\s' + _symbol = r'(?u)(?:\|[^|]*\||\\[\w\W]|[^|\\%s]+)+' % _delimiters + _number_prefix = r'(?:#e)?(?:#b|(?:#d)?)(?:#e)?' + _exponent = r'(?:[defls][-+]?\d+)' + _inexact_simple_no_hashes = r'(?:\d+(?:/\d+|\.\d*)?|\.\d+)' + _inexact_simple = (r'(?:%s|(?:\d+#+(?:\.#*|/\d+#*)?|\.\d+#+|' + r'\d+(?:\.\d*#+|/\d+#+)))' % _inexact_simple_no_hashes) + _inexact_normal_no_hashes = r'(?:%s%s?)' % (_inexact_simple_no_hashes, + _exponent) + _inexact_normal = r'(?:%s%s?)' % (_inexact_simple, _exponent) + _inexact_special = r'(?:(?:inf|nan)\.[0f])' + _inexact_real = r'(?:[-+]?%s|[-+]%s)' % (_inexact_normal, + _inexact_special) + _inexact_unsigned = r'(?:%s|%s)' % (_inexact_normal, _inexact_special) tokens = { - 'root' : [ - (r';.*$', Comment.Single), - (r'#\|[^|]+\|#', Comment.Multiline), - - # whitespaces - usually not relevant - (r'\s+', Text), - - ## numbers: Keep in mind Racket reader hash prefixes, - ## which can denote the base or the type. These don't map - ## neatly onto pygments token types; some judgment calls - ## here. Note that none of these regexps attempt to - ## exclude identifiers that start with a number, such as a - ## variable named "100-Continue". - - # #b - (r'#b[-+]?[01]+\.[01]+', Number.Float), - (r'#b[01]+e[-+]?[01]+', Number.Float), - (r'#b[-+]?[01]/[01]+', Number), - (r'#b[-+]?[01]+', Number.Integer), - (r'#b\S*', Error), - - # #d OR no hash prefix - (r'(#d)?[-+]?\d+\.\d+', Number.Float), - (r'(#d)?\d+e[-+]?\d+', Number.Float), - (r'(#d)?[-+]?\d+/\d+', Number), - (r'(#d)?[-+]?\d+', Number.Integer), - (r'#d\S*', Error), - - # #e - (r'#e[-+]?\d+\.\d+', Number.Float), - (r'#e\d+e[-+]?\d+', Number.Float), - (r'#e[-+]?\d+/\d+', Number), - (r'#e[-+]?\d+', Number), - (r'#e\S*', Error), - - # #i is always inexact-real, i.e. float - (r'#i[-+]?\d+\.\d+', Number.Float), - (r'#i\d+e[-+]?\d+', Number.Float), - (r'#i[-+]?\d+/\d+', Number.Float), - (r'#i[-+]?\d+', Number.Float), - (r'#i\S*', Error), + 'root': [ + (_closing_parenthesis, Error), + (r'(?!\Z)', Text, 'unquoted-datum') + ], + 'datum': [ + (r'(?s)#;|#*', Comment), + (u';[^\\n\\r\x85\u2028\u2029]*', Comment.Single), + (r'#\|', Comment.Multiline, 'block-comment'), + + # Whitespaces + (r'(?u)\s+', Text), + + # Numbers: Keep in mind Racket reader hash prefixes, which + # can denote the base or the type. These don't map neatly + # onto Pygments token types; some judgment calls here. + + # #b or #d or no prefix + (r'(?i)%s[-+]?\d+(?=[%s])' % (_number_prefix, _delimiters), + Number.Integer, '#pop'), + (r'(?i)%s[-+]?(\d+(\.\d*)?|\.\d+)([deflst][-+]?\d+)?(?=[%s])' % + (_number_prefix, _delimiters), Number.Float, '#pop'), + (r'(?i)%s[-+]?(%s([-+]%s?i)?|[-+]%s?i)(?=[%s])' % + (_number_prefix, _inexact_normal_no_hashes, + _inexact_normal_no_hashes, _inexact_normal_no_hashes, + _delimiters), Number, '#pop'), + + # Inexact without explicit #i + (r'(?i)(#[bd])?(%s([-+]%s?i)?|[-+]%s?i|%s@%s)(?=[%s])' % + (_inexact_real, _inexact_unsigned, _inexact_unsigned, + _inexact_real, _inexact_real, _delimiters), Number.Float, + '#pop'), + + # The remaining extflonums + (r'(?i)(([-+]?%st[-+]?\d+)|[-+](inf|nan)\.t)(?=[%s])' % + (_inexact_simple, _delimiters), Number.Float, '#pop'), # #o - (r'#o[-+]?[0-7]+\.[0-7]+', Number.Oct), - (r'#o[0-7]+e[-+]?[0-7]+', Number.Oct), - (r'#o[-+]?[0-7]+/[0-7]+', Number.Oct), - (r'#o[-+]?[0-7]+', Number.Oct), - (r'#o\S*', Error), + (r'(?i)(#[ei])?#o%s' % _symbol, Number.Oct, '#pop'), # #x - (r'#x[-+]?[0-9a-fA-F]+\.[0-9a-fA-F]+', Number.Hex), - # the exponent variation (e.g. #x1e1) is N/A - (r'#x[-+]?[0-9a-fA-F]+/[0-9a-fA-F]+', Number.Hex), - (r'#x[-+]?[0-9a-fA-F]+', Number.Hex), - (r'#x\S*', Error), - - - # strings, symbols and characters - (r'"(\\\\|\\"|[^"])*"', String), - (r"'" + valid_name, String.Symbol), - (r"#\\([()/'\"._!§$%& ?=+-]{1}|[a-zA-Z0-9]+)", String.Char), - (r'#rx".+"', String.Regex), - (r'#px".+"', String.Regex), - - # constants - (r'(#t|#f)', Name.Constant), - - # keyword argument names (e.g. #:keyword) - (r'#:\S+', Keyword.Declaration), - - # #lang - (r'#lang \S+', Keyword.Namespace), - - # special operators - (r"('|#|`|,@|,|\.)", Operator), - - # highlight the keywords - ('(%s)' % '|'.join([ - re.escape(entry) + ' ' for entry in keywords]), - Keyword - ), - - # first variable in a quoted string like - # '(this is syntactic sugar) - (r"(?<='\()" + valid_name, Name.Variable), - (r"(?<=#\()" + valid_name, Name.Variable), - - # highlight the builtins - ("(?<=\()(%s)" % '|'.join([ - re.escape(entry) + ' ' for entry in builtins]), - Name.Builtin - ), - - # the remaining functions; handle both ( and [ - (r'(?<=(\(|\[|\{))' + valid_name, Name.Function), - - # find the remaining variables - (valid_name, Name.Variable), - - # the famous parentheses! - (r'(\(|\)|\[|\]|\{|\})', Punctuation), + (r'(?i)(#[ei])?#x%s' % _symbol, Number.Hex, '#pop'), + + # #i is always inexact, i.e. float + (r'(?i)(#[bd])?#i%s' % _symbol, Number.Float, '#pop'), + + # Strings and characters + (r'#?"', String.Double, ('#pop', 'string')), + (r'#<<(.+)\n(^(?!\1$).*$\n)*^\1$', String.Heredoc, '#pop'), + (r'#\\(u[\da-fA-F]{1,4}|U[\da-fA-F]{1,8})', String.Char, '#pop'), + (r'(?is)#\\([0-7]{3}|[a-z]+|.)', String.Char, '#pop'), + (r'(?s)#[pr]x#?"(\\?.)*?"', String.Regex, '#pop'), + + # Constants + (r'#(true|false|[tTfF])', Name.Constant, '#pop'), + + # Keyword argument names (e.g. #:keyword) + (r'#:%s' % _symbol, Keyword.Declaration, '#pop'), + + # Reader extensions + (r'(#lang |#!)(\S+)', + bygroups(Keyword.Namespace, Name.Namespace)), + (r'#reader', Keyword.Namespace, 'quoted-datum'), + + # Other syntax + (r"(?i)\.(?=[%s])|#c[is]|#['`]|#,@?" % _delimiters, Operator), + (r"'|#[s&]|#hash(eqv?)?|#\d*(?=%s)" % _opening_parenthesis, + Operator, ('#pop', 'quoted-datum')) + ], + 'datum*': [ + (r'`|,@?', Operator), + (_symbol, String.Symbol, '#pop'), + (r'[|\\]', Error), + (r'', Text, '#pop') + ], + 'list': [ + (_closing_parenthesis, Punctuation, '#pop') + ], + 'unquoted-datum': [ + include('datum'), + (r'quote(?=[%s])' % _delimiters, Keyword, + ('#pop', 'quoted-datum')), + (r'`', Operator, ('#pop', 'quasiquoted-datum')), + (r'quasiquote(?=[%s])' % _delimiters, Keyword, + ('#pop', 'quasiquoted-datum')), + (_opening_parenthesis, Punctuation, ('#pop', 'unquoted-list')), + (r'(?u)(%s)(?=[%s])' % ('|'.join( + [re.escape(entry) for entry in _keywords]), _delimiters), + Keyword, '#pop'), + (r'(?u)(%s)(?=[%s])' % ('|'.join( + [re.escape(entry) for entry in _builtins]), _delimiters), + Name.Builtin, '#pop'), + (_symbol, Name, '#pop'), + include('datum*') + ], + 'unquoted-list': [ + include('list'), + (r'(?!\Z)', Text, 'unquoted-datum') + ], + 'quasiquoted-datum': [ + include('datum'), + (r',@?', Operator, ('#pop', 'unquoted-datum')), + (r'unquote(-splicing)?(?=[%s])' % _delimiters, Keyword, + ('#pop', 'unquoted-datum')), + (_opening_parenthesis, Punctuation, ('#pop', 'quasiquoted-list')), + include('datum*') + ], + 'quasiquoted-list': [ + include('list'), + (r'(?!\Z)', Text, 'quasiquoted-datum') + ], + 'quoted-datum': [ + include('datum'), + (_opening_parenthesis, Punctuation, ('#pop', 'quoted-list')), + include('datum*') + ], + 'quoted-list': [ + include('list'), + (r'(?!\Z)', Text, 'quoted-datum') + ], + 'block-comment': [ + (r'#\|', Comment.Multiline, '#push'), + (r'\|#', Comment.Multiline, '#pop'), + (r'[^#|]+|.', Comment.Multiline) ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'(?s)\\([0-7]{1,3}|x[\da-fA-F]{1,2}|u[\da-fA-F]{1,4}|' + r'U[\da-fA-F]{1,8}|.)', String.Escape), + (r'[^\\"]+', String.Double) + ] } @@ -732,7 +1066,7 @@ class CommonLispLexer(RegexLexer): ### couple of useful regexes # characters that are not macro-characters and can be used to begin a symbol - nonmacro = r'\\.|[a-zA-Z0-9!$%&*+-/<=>?@\[\]^_{}~]' + nonmacro = r'\\.|[\w!$%&*+-/<=>?@\[\]^{}~]' constituent = nonmacro + '|[#.:]' terminated = r'(?=[ "()\'\n,;`])' # whitespace or terminating macro characters @@ -850,10 +1184,10 @@ class CommonLispLexer(RegexLexer): (r'#[oO][+-]?[0-7]+(/[0-7]+)?', Number.Oct), # hex rational - (r'#[xX][+-]?[0-9a-fA-F]+(/[0-9a-fA-F]+)?', Number.Hex), + (r'#[xX][+-]?[0-9a-f]+(/[0-9a-f]+)?', Number.Hex), # radix rational - (r'#\d+[rR][+-]?[0-9a-zA-Z]+(/[0-9a-zA-Z]+)?', Number), + (r'#\d+[rR][+-]?[0-9a-z]+(/[0-9a-z]+)?', Number), # complex (r'(#[cC])(\()', bygroups(Number, Punctuation), 'body'), @@ -894,6 +1228,142 @@ class CommonLispLexer(RegexLexer): } +class CryptolLexer(RegexLexer): + """ + FIXME: A Cryptol2 lexer based on the lexemes defined in the Haskell 98 Report. + + .. versionadded:: 2.0 + """ + name = 'Cryptol' + aliases = ['cryptol', 'cry'] + filenames = ['*.cry'] + mimetypes = ['text/x-cryptol'] + + reserved = ['Arith','Bit','Cmp','False','Inf','True','else', + 'export','extern','fin','if','import','inf','lg2', + 'max','min','module','newtype','pragma','property', + 'then','type','where','width'] + ascii = ['NUL','SOH','[SE]TX','EOT','ENQ','ACK', + 'BEL','BS','HT','LF','VT','FF','CR','S[OI]','DLE', + 'DC[1-4]','NAK','SYN','ETB','CAN', + 'EM','SUB','ESC','[FGRU]S','SP','DEL'] + + tokens = { + 'root': [ + # Whitespace: + (r'\s+', Text), + #(r'--\s*|.*$', Comment.Doc), + (r'//.*$', Comment.Single), + (r'/\*', Comment.Multiline, 'comment'), + # Lexemes: + # Identifiers + (r'\bimport\b', Keyword.Reserved, 'import'), + (r'\bmodule\b', Keyword.Reserved, 'module'), + (r'\berror\b', Name.Exception), + (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), + (r'^[_a-z][\w\']*', Name.Function), + (r"'?[_a-z][\w']*", Name), + (r"('')?[A-Z][\w\']*", Keyword.Type), + # Operators + (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator + (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials + (r':[:!#$%&*+.\\/<=>?@^|~-]*', Keyword.Type), # Constructor operators + (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), # Other operators + # Numbers + (r'\d+[eE][+-]?\d+', Number.Float), + (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), + (r'0[oO][0-7]+', Number.Oct), + (r'0[xX][\da-fA-F]+', Number.Hex), + (r'\d+', Number.Integer), + # Character/String Literals + (r"'", String.Char, 'character'), + (r'"', String, 'string'), + # Special + (r'\[\]', Keyword.Type), + (r'\(\)', Name.Builtin), + (r'[][(),;`{}]', Punctuation), + ], + 'import': [ + # Import statements + (r'\s+', Text), + (r'"', String, 'string'), + # after "funclist" state + (r'\)', Punctuation, '#pop'), + (r'qualified\b', Keyword), + # import X as Y + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(as)(\s+)([A-Z][a-zA-Z0-9_.]*)', + bygroups(Name.Namespace, Text, Keyword, Text, Name), '#pop'), + # import X hiding (functions) + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(hiding)(\s+)(\()', + bygroups(Name.Namespace, Text, Keyword, Text, Punctuation), 'funclist'), + # import X (functions) + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + bygroups(Name.Namespace, Text, Punctuation), 'funclist'), + # import X + (r'[a-zA-Z0-9_.]+', Name.Namespace, '#pop'), + ], + 'module': [ + (r'\s+', Text), + (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + bygroups(Name.Namespace, Text, Punctuation), 'funclist'), + (r'[A-Z][a-zA-Z0-9_.]*', Name.Namespace, '#pop'), + ], + 'funclist': [ + (r'\s+', Text), + (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type), + (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), + (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single), + (r'{-', Comment.Multiline, 'comment'), + (r',', Punctuation), + (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), + # (HACK, but it makes sense to push two instances, believe me) + (r'\(', Punctuation, ('funclist', 'funclist')), + (r'\)', Punctuation, '#pop:2'), + ], + 'comment': [ + # Multiline Comments + (r'[^/\*]+', Comment.Multiline), + (r'/\*', Comment.Multiline, '#push'), + (r'\*/', Comment.Multiline, '#pop'), + (r'[\*/]', Comment.Multiline), + ], + 'character': [ + # Allows multi-chars, incorrectly. + (r"[^\\']'", String.Char, '#pop'), + (r"\\", String.Escape, 'escape'), + ("'", String.Char, '#pop'), + ], + 'string': [ + (r'[^\\"]+', String), + (r"\\", String.Escape, 'escape'), + ('"', String, '#pop'), + ], + 'escape': [ + (r'[abfnrtv"\'&\\]', String.Escape, '#pop'), + (r'\^[][A-Z@\^_]', String.Escape, '#pop'), + ('|'.join(ascii), String.Escape, '#pop'), + (r'o[0-7]+', String.Escape, '#pop'), + (r'x[\da-fA-F]+', String.Escape, '#pop'), + (r'\d+', String.Escape, '#pop'), + (r'\s+\\', String.Escape, '#pop'), + ], + } + + EXTRA_KEYWORDS = ['join', 'split', 'reverse', 'transpose', 'width', + 'length', 'tail', '<<', '>>', '<<<', '>>>', 'const', + 'reg', 'par', 'seq', 'ASSERT', 'undefined', 'error', + 'trace'] + + def get_tokens_unprocessed(self, text): + stack = ['root'] + for index, token, value in \ + RegexLexer.get_tokens_unprocessed(self, text, stack): + if token is Name and value in self.EXTRA_KEYWORDS: + yield index, Name.Builtin, value + else: + yield index, token, value + + class HaskellLexer(RegexLexer): """ A Haskell lexer based on the lexemes defined in the Haskell 98 Report. @@ -905,6 +1375,8 @@ class HaskellLexer(RegexLexer): filenames = ['*.hs'] mimetypes = ['text/x-haskell'] + flags = re.MULTILINE | re.UNICODE + reserved = ['case','class','data','default','deriving','do','else', 'if','in','infix[lr]?','instance', 'let','newtype','of','then','type','where','_'] @@ -926,9 +1398,9 @@ class HaskellLexer(RegexLexer): (r'\bmodule\b', Keyword.Reserved, 'module'), (r'\berror\b', Name.Exception), (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), - (r'^[_a-z][\w\']*', Name.Function), - (r"'?[_a-z][\w']*", Name), - (r"('')?[A-Z][\w\']*", Keyword.Type), + (r'^[_' + uni.Ll + r'][\w\']*', Name.Function), + (r"'?[_" + uni.Ll + r"'][\w']*", Name), + (r"('')?[" + uni.Lu + r"][\w\']*", Keyword.Type), # Operators (r'\\(?![:!#$%&*+.\\/<=>?@^|~-]+)', Name.Function), # lambda operator (r'(<-|::|->|=>|=)(?![:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials @@ -956,28 +1428,28 @@ class HaskellLexer(RegexLexer): (r'\)', Punctuation, '#pop'), (r'qualified\b', Keyword), # import X as Y - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(as)(\s+)([A-Z][a-zA-Z0-9_.]*)', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(as)(\s+)([' + uni.Lu + r'][\w.]*)', bygroups(Name.Namespace, Text, Keyword, Text, Name), '#pop'), # import X hiding (functions) - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(hiding)(\s+)(\()', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(hiding)(\s+)(\()', bygroups(Name.Namespace, Text, Keyword, Text, Punctuation), 'funclist'), # import X (functions) - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()', bygroups(Name.Namespace, Text, Punctuation), 'funclist'), # import X - (r'[a-zA-Z0-9_.]+', Name.Namespace, '#pop'), + (r'[\w.]+', Name.Namespace, '#pop'), ], 'module': [ (r'\s+', Text), - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + (r'([' + uni.Lu + r'][\w.]*)(\s+)(\()', bygroups(Name.Namespace, Text, Punctuation), 'funclist'), - (r'[A-Z][a-zA-Z0-9_.]*', Name.Namespace, '#pop'), + (r'[' + uni.Lu + r'][\w.]*', Name.Namespace, '#pop'), ], 'funclist': [ (r'\s+', Text), - (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type), - (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), - (r'--.*$', Comment.Single), + (r'[' + uni.Lu + r']\w*', Keyword.Type), + (r'(_[\w\']+|[' + uni.Ll + r'][\w\']*)', Name.Function), + (r'--(?![!#$%&*+./<=>?@\^|_~:\\]).*?$', Comment.Single), (r'{-', Comment.Multiline, 'comment'), (r',', Punctuation), (r'[:!#$%&*+.\\/<=>?@^|~-]+', Operator), @@ -1007,7 +1479,7 @@ class HaskellLexer(RegexLexer): ], 'escape': [ (r'[abfnrtv"\'&\\]', String.Escape, '#pop'), - (r'\^[][A-Z@\^_]', String.Escape, '#pop'), + (r'\^[][' + uni.Lu + r'@\^_]', String.Escape, '#pop'), ('|'.join(ascii), String.Escape, '#pop'), (r'o[0-7]+', String.Escape, '#pop'), (r'x[\da-fA-F]+', String.Escape, '#pop'), @@ -1061,7 +1533,7 @@ class IdrisLexer(RegexLexer): (r'\b(%s)(?!\')\b' % '|'.join(reserved), Keyword.Reserved), (r'(import|module)(\s+)', bygroups(Keyword.Reserved, Text), 'module'), (r"('')?[A-Z][\w\']*", Keyword.Type), - (r'[a-z][A-Za-z0-9_\']*', Text), + (r'[a-z][\w\']*', Text), # Special Symbols (r'(<-|::|->|=>|=)', Operator.Word), # specials (r'([\(\)\{\}\[\]:!#$%&*+.\\/<=>?@^|~-]+)', Operator.Word), # specials @@ -1078,13 +1550,13 @@ class IdrisLexer(RegexLexer): ], 'module': [ (r'\s+', Text), - (r'([A-Z][a-zA-Z0-9_.]*)(\s+)(\()', + (r'([A-Z][\w.]*)(\s+)(\()', bygroups(Name.Namespace, Text, Punctuation), 'funclist'), - (r'[A-Z][a-zA-Z0-9_.]*', Name.Namespace, '#pop'), + (r'[A-Z][\w.]*', Name.Namespace, '#pop'), ], 'funclist': [ (r'\s+', Text), - (r'[A-Z][a-zA-Z0-9_]*', Keyword.Type), + (r'[A-Z]\w*', Keyword.Type), (r'(_[\w\']+|[a-z][\w\']*)', Name.Function), (r'--.*$', Comment.Single), (r'{-', Comment.Multiline, 'comment'), @@ -1163,7 +1635,7 @@ class AgdaLexer(RegexLexer): (r'\b(Set|Prop)\b', Keyword.Type), # Special Symbols (r'(\(|\)|\{|\})', Operator), - (u'(\\.{1,3}|\\||[\u039B]|[\u2200]|[\u2192]|:|=|->)', Operator.Word), + (u'(\\.{1,3}|\\||\u039B|\u2200|\u2192|:|=|->)', Operator.Word), # Numbers (r'\d+[eE][+-]?\d+', Number.Float), (r'\d+\.\d+([eE][+-]?\d+)?', Number.Float), @@ -1184,7 +1656,7 @@ class AgdaLexer(RegexLexer): ], 'module': [ (r'{-', Comment.Multiline, 'comment'), - (r'[a-zA-Z][a-zA-Z0-9_.]*', Name, '#pop'), + (r'[a-zA-Z][\w.]*', Name, '#pop'), (r'[^a-zA-Z]*', Text) ], 'comment': HaskellLexer.tokens['comment'], @@ -1259,6 +1731,29 @@ class LiterateLexer(Lexer): yield item +class LiterateCryptolLexer(LiterateLexer): + """ + For Literate Cryptol (Bird-style or LaTeX) source. + + Additional options accepted: + + `litstyle` + If given, must be ``"bird"`` or ``"latex"``. If not given, the style + is autodetected: if the first non-whitespace character in the source + is a backslash or percent character, LaTeX is assumed, else Bird. + + .. versionadded:: 0.9 + """ + name = 'Literate Cryptol' + aliases = ['lcry', 'literate-cryptol', 'lcryptol'] + filenames = ['*.lcry'] + mimetypes = ['text/x-literate-cryptol'] + + def __init__(self, **options): + crylexer = CryptolLexer(**options) + LiterateLexer.__init__(self, crylexer, **options) + + class LiterateHaskellLexer(LiterateLexer): """ For Literate Haskell (Bird-style or LaTeX) source. @@ -1360,7 +1855,7 @@ class SMLLexer(RegexLexer): nonid_reserved = [ '(', ')', '[', ']', '{', '}', ',', ';', '...', '_' ] - alphanumid_re = r"[a-zA-Z][a-zA-Z0-9_']*" + alphanumid_re = r"[a-zA-Z][\w']*" symbolicid_re = r"[!%&$#+\-/:<=>?@\\~`^|*]+" # A character constant is a sequence of the form #s, where s is a string @@ -1450,7 +1945,7 @@ class SMLLexer(RegexLexer): (r'\b(type|eqtype)\b(?!\')', Keyword.Reserved, 'tname'), # Regular identifiers, long and otherwise - (r'\'[0-9a-zA-Z_\']*', Name.Decorator), + (r'\'[\w\']*', Name.Decorator), (r'(%s)(\.)' % alphanumid_re, long_id_callback, "dotted"), (r'(%s)' % alphanumid_re, id_callback), (r'(%s)' % symbolicid_re, id_callback), @@ -1697,9 +2192,9 @@ class OcamlLexer(RegexLexer): 'root': [ (r'\s+', Text), (r'false|true|\(\)|\[\]', Name.Builtin.Pseudo), - (r'\b([A-Z][A-Za-z0-9_\']*)(?=\s*\.)', + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), - (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class), + (r'\b([A-Z][\w\']*)', Name.Class), (r'\(\*(?![)])', Comment, 'comment'), (r'\b(%s)\b' % '|'.join(keywords), Keyword), (r'(%s)' % '|'.join(keyopts[::-1]), Operator), @@ -1739,9 +2234,9 @@ class OcamlLexer(RegexLexer): 'dotted': [ (r'\s+', Text), (r'\.', Punctuation), - (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace), - (r'[A-Z][A-Za-z0-9_\']*', Name.Class, '#pop'), - (r'[a-z_][A-Za-z0-9_\']*', Name, '#pop'), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), + (r'[a-z_][\w\']*', Name, '#pop'), ], } @@ -1801,9 +2296,9 @@ class ErlangLexer(RegexLexer): 'div', 'not', 'or', 'orelse', 'rem', 'xor' ] - atom_re = r"(?:[a-z][a-zA-Z0-9_]*|'[^\n']*[^\\]')" + atom_re = r"(?:[a-z]\w*|'[^\n']*[^\\]')" - variable_re = r'(?:[A-Z_][a-zA-Z0-9_]*)' + variable_re = r'(?:[A-Z_]\w*)' escape_re = r'(?:\\(?:[bdefnrstv\'"\\/]|[0-7][0-7]?[0-7]?|\^[a-zA-Z]))' @@ -2309,9 +2804,9 @@ class CoqLexer(RegexLexer): (r'\b(%s)\b' % '|'.join(keywords4), Keyword), (r'\b(%s)\b' % '|'.join(keywords5), Keyword.Pseudo), (r'\b(%s)\b' % '|'.join(keywords6), Keyword.Reserved), - (r'\b([A-Z][A-Za-z0-9_\']*)(?=\s*\.)', + (r'\b([A-Z][\w\']*)(?=\s*\.)', Name.Namespace, 'dotted'), - (r'\b([A-Z][A-Za-z0-9_\']*)', Name.Class), + (r'\b([A-Z][\w\']*)', Name.Class), (r'(%s)' % '|'.join(keyopts[::-1]), Operator), (r'(%s|%s)?%s' % (infix_syms, prefix_syms, operators), Operator), (r'\b(%s)\b' % '|'.join(word_operators), Operator.Word), @@ -2348,8 +2843,8 @@ class CoqLexer(RegexLexer): 'dotted': [ (r'\s+', Text), (r'\.', Punctuation), - (r'[A-Z][A-Za-z0-9_\']*(?=\s*\.)', Name.Namespace), - (r'[A-Z][A-Za-z0-9_\']*', Name.Class, '#pop'), + (r'[A-Z][\w\']*(?=\s*\.)', Name.Namespace), + (r'[A-Z][\w\']*', Name.Class, '#pop'), (r'[a-z][a-z0-9_\']*', Name, '#pop'), (r'', Text, '#pop') ], @@ -2437,7 +2932,7 @@ class NewLispLexer(RegexLexer): ] # valid names - valid_name = r'([a-zA-Z0-9!$%&*+.,/<=>?@^_~|-])+|(\[.*?\])+' + valid_name = r'([\w!$%&*+.,/<=>?@^~|-])+|(\[.*?\])+' tokens = { 'root': [ @@ -2552,15 +3047,15 @@ class NixLexer(RegexLexer): (r"''", String.Single, 'singlequote'), # paths - (r'[a-zA-Z0-9._+-]*(\/[a-zA-Z0-9._+-]+)+', Literal), - (r'\<[a-zA-Z0-9._+-]+(\/[a-zA-Z0-9._+-]+)*\>', Literal), + (r'[\w.+-]*(\/[\w.+-]+)+', Literal), + (r'\<[\w.+-]+(\/[\w.+-]+)*\>', Literal), # urls - (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[a-zA-Z0-9%/?:@&=+$,\\_.!~*\'-]+', Literal), + (r'[a-zA-Z][a-zA-Z0-9\+\-\.]*\:[\w%/?:@&=+$,\\.!~*\'-]+', Literal), # names of variables - (r'[a-zA-Z0-9-_]+\s*=', String.Symbol), - (r'[a-zA-Z_][a-zA-Z0-9_\'-]*', Text), + (r'[\w-]+\s*=', String.Symbol), + (r'[a-zA-Z_][\w\'-]*', Text), ], 'comment': [ @@ -2679,7 +3174,7 @@ class ElixirLexer(RegexLexer): (r':"', String.Symbol, 'interpoling_symbol'), (r'\b(nil|true|false)\b(?![?!])|\b[A-Z]\w*\b', Name.Constant), (r'\b(__(FILE|LINE|MODULE|MAIN|FUNCTION)__)\b(?![?!])', Name.Builtin.Pseudo), - (r'[a-zA-Z_!][\w_]*[!\?]?', Name), + (r'[a-zA-Z_!]\w*[!\?]?', Name), (r'[(){};,/\|:\\\[\]]', Punctuation), (r'@[a-zA-Z_]\w*|&\d', Name.Variable), (r'\b(0[xX][0-9A-Fa-f]+|\d(_?\d)*(\.(?![^\d\s])' diff --git a/pygments/lexers/graph.py b/pygments/lexers/graph.py new file mode 100644 index 00000000..6aa446c7 --- /dev/null +++ b/pygments/lexers/graph.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.graph + ~~~~~~~~~~~~~~~~~~~~~ + + Lexers for graph query languages. + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups, using, this +from pygments.token import Keyword, Punctuation, Text, Comment, Operator, Name,\ + String, Number, Generic, Whitespace + + +__all__ = ['CypherLexer'] + + +class CypherLexer(RegexLexer): + """ + For `Cypher Query Language + <http://docs.neo4j.org/chunked/milestone/cypher-query-lang.html>`_ + + For the Cypher version in Neo4J 2.0 + + .. versionadded:: 2.0 + """ + name = 'Cypher' + aliases = ['cypher'] + filenames = ['*.cyp','*.cypher'] + + flags = re.MULTILINE | re.IGNORECASE + + tokens = { + 'root': [ + include('comment'), + include('keywords'), + include('clauses'), + include('relations'), + include('strings'), + include('whitespace'), + include('barewords'), + ], + 'comment': [ + (r'^.*//.*\n', Comment.Single), + ], + 'keywords': [ + (r'(create|order|match|limit|set|skip|start|return|with|where|' + r'delete|foreach|not|by)\b', Keyword), + ], + 'clauses': [ + # TODO: many missing ones, see http://docs.neo4j.org/refcard/2.0/ + (r'(all|any|as|asc|create|create\s+unique|delete|' + r'desc|distinct|foreach|in|is\s+null|limit|match|none|' + r'order\s+by|return|set|skip|single|start|union|where|with)\b', + Keyword), + ], + 'relations': [ + (r'(-\[)(.*?)(\]->)', bygroups(Operator, using(this), Operator)), + (r'(<-\[)(.*?)(\]-)', bygroups(Operator, using(this), Operator)), + (r'-->|<--|\[|\]', Operator), + (r'<|>|<>|=|<=|=>|\(|\)|\||:|,|;', Punctuation), + (r'[.*{}]', Punctuation), + ], + 'strings': [ + (r'"(?:\\[tbnrf\'\"\\]|[^\\"])*"', String), + (r'`(?:``|[^`])+`', Name.Variable), + ], + 'whitespace': [ + (r'\s+', Whitespace), + ], + 'barewords': [ + (r'[a-z]\w*', Name), + (r'\d+', Number), + ], + } + + diff --git a/pygments/lexers/hdl.py b/pygments/lexers/hdl.py index 1ebe4e5c..6240cfe0 100644 --- a/pygments/lexers/hdl.py +++ b/pygments/lexers/hdl.py @@ -54,7 +54,7 @@ class VerilogLexer(RegexLexer): (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.;\']', Punctuation), - (r'`[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), + (r'`[a-zA-Z_]\w*', Name.Constant), (r'^(\s*)(package)(\s+)', bygroups(Text, Keyword.Namespace, Text)), (r'^(\s*)(import)(\s+)', bygroups(Text, Keyword.Namespace, Text), @@ -96,8 +96,8 @@ class VerilogLexer(RegexLexer): r'bit|logic|reg|' r'supply0|supply1|tri|triand|trior|tri0|tri1|trireg|uwire|wire|wand|wor' r'shortreal|real|realtime)\b', Keyword.Type), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*:(?!:)', Name.Label), + ('[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -115,7 +115,7 @@ class VerilogLexer(RegexLexer): (r'\n', Comment.Preproc, '#pop'), ], 'import': [ - (r'[a-zA-Z0-9_:]+\*?', Name.Namespace, '#pop') + (r'[\w:]+\*?', Name.Namespace, '#pop') ] } @@ -169,7 +169,7 @@ class SystemVerilogLexer(RegexLexer): (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]', Operator), (r'[()\[\],.;\']', Punctuation), - (r'`[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), + (r'`[a-zA-Z_]\w*', Name.Constant), (r'(accept_on|alias|always|always_comb|always_ff|always_latch|' r'and|assert|assign|assume|automatic|before|begin|bind|bins|' @@ -230,11 +230,11 @@ class SystemVerilogLexer(RegexLexer): r'bit|logic|reg|' r'supply0|supply1|tri|triand|trior|tri0|tri1|trireg|uwire|wire|wand|wor' r'shortreal|real|realtime)\b', Keyword.Type), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*:(?!:)', Name.Label), + ('[a-zA-Z_]\w*', Name), ], 'classname': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop'), + (r'[a-zA-Z_]\w*', Name.Class, '#pop'), ], 'string': [ (r'"', String, '#pop'), @@ -252,7 +252,7 @@ class SystemVerilogLexer(RegexLexer): (r'\n', Comment.Preproc, '#pop'), ], 'import': [ - (r'[a-zA-Z0-9_:]+\*?', Name.Namespace, '#pop') + (r'[\w:]+\*?', Name.Namespace, '#pop') ] } @@ -290,19 +290,19 @@ class VhdlLexer(RegexLexer): (r'--(?![!#$%&*+./<=>?@\^|_~]).*?$', Comment.Single), (r"'(U|X|0|1|Z|W|L|H|-)'", String.Char), (r'[~!%^&*+=|?:<>/-]', Operator), - (r"'[a-zA-Z_][a-zA-Z0-9_]*", Name.Attribute), + (r"'[a-z_]\w*", Name.Attribute), (r'[()\[\],.;\']', Punctuation), (r'"[^\n\\]*"', String), - (r'(library)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(library)(\s+)([a-z_]\w*)', bygroups(Keyword, Text, Name.Namespace)), (r'(use)(\s+)(entity)', bygroups(Keyword, Text, Keyword)), - (r'(use)(\s+)([a-zA-Z_][\.a-zA-Z0-9_]*)', + (r'(use)(\s+)([a-z_][\.\w]*)', bygroups(Keyword, Text, Name.Namespace)), - (r'(entity|component)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(entity|component)(\s+)([a-z_]\w*)', bygroups(Keyword, Text, Name.Class)), - (r'(architecture|configuration)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)(\s+)' - r'(of)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)(\s+)(is)', + (r'(architecture|configuration)(\s+)([a-z_]\w*)(\s+)' + r'(of)(\s+)([a-z_]\w*)(\s+)(is)', bygroups(Keyword, Text, Name.Class, Text, Keyword, Text, Name.Class, Text, Keyword)), @@ -312,11 +312,11 @@ class VhdlLexer(RegexLexer): include('keywords'), include('numbers'), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), ], 'endblock': [ include('keywords'), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class), + (r'[a-z_]\w*', Name.Class), (r'(\s+)', Text), (r';', Punctuation, '#pop'), ], @@ -345,11 +345,11 @@ class VhdlLexer(RegexLexer): r'while|with|xnor|xor)\b', Keyword), ], 'numbers': [ - (r'\d{1,2}#[0-9a-fA-F_]+#?', Number.Integer), + (r'\d{1,2}#[0-9a-f_]+#?', Number.Integer), (r'[0-1_]+(\.[0-1_])', Number.Integer), (r'\d+', Number.Integer), (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+', Number.Float), - (r'H"[0-9a-fA-F_]+"', Number.Oct), + (r'H"[0-9a-f_]+"', Number.Oct), (r'O"[0-7_]+"', Number.Oct), (r'B"[0-1_]+"', Number.Oct), ], diff --git a/pygments/lexers/inferno.py b/pygments/lexers/inferno.py index 56f02b98..6aecafdb 100644 --- a/pygments/lexers/inferno.py +++ b/pygments/lexers/inferno.py @@ -35,7 +35,7 @@ class LimboLexer(RegexLexer): tokens = { 'whitespace': [ - (r'^(\s*)([a-zA-Z_][a-zA-Z0-9_]*:(\s*)\n)', + (r'^(\s*)([a-zA-Z_]\w*:(\s*)\n)', bygroups(Text, Name.Label)), (r'\n', Text), (r'\s+', Text), @@ -64,7 +64,7 @@ class LimboLexer(RegexLexer): (r'(byte|int|big|real|string|array|chan|list|adt' r'|fn|ref|of|module|self|type)\b', Keyword.Type), (r'(con|iota|nil)\b', Keyword.Constant), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'statement' : [ include('whitespace'), diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index 6d8a2584..e9c9be20 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -21,7 +21,7 @@ from pygments import unistring as uni __all__ = ['JavaLexer', 'ScalaLexer', 'GosuLexer', 'GosuTemplateLexer', 'GroovyLexer', 'IokeLexer', 'ClojureLexer', 'ClojureScriptLexer', 'KotlinLexer', 'XtendLexer', 'AspectJLexer', 'CeylonLexer', - 'PigLexer', 'GoloLexer'] + 'PigLexer', 'GoloLexer', 'JasminLexer'] class JavaLexer(RegexLexer): @@ -34,14 +34,19 @@ class JavaLexer(RegexLexer): filenames = ['*.java'] mimetypes = ['text/x-java'] - flags = re.MULTILINE | re.DOTALL + flags = re.MULTILINE | re.DOTALL | re.UNICODE tokens = { 'root': [ (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + # method names + (r'((?:(?:[^\W\d]|\$)[\w\.\[\]\$<>]*\s+)+?)' # return arguments + r'((?:[^\W\d]|\$)[\w\$]*)' # method name + r'(\s*)(\()', # signature start + bygroups(using(this), Name.Function, Text, Operator)), + (r'@[^\W\d][\w\.]*', Name.Decorator), (r'(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|goto|instanceof|new|return|switch|this|throw|try|while)\b', Keyword), @@ -50,20 +55,15 @@ class JavaLexer(RegexLexer): r'transient|volatile)\b', Keyword.Declaration), (r'(boolean|byte|char|double|float|int|long|short|void)\b', Keyword.Type), - # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]<>]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start - bygroups(using(this), Name.Function, Text, Operator)), (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)), (r'(true|false|null)\b', Keyword.Constant), (r'(class|interface)(\s+)', bygroups(Keyword.Declaration, Text), 'class'), (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), (r'"(\\\\|\\"|[^"])*"', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'(\.)((?:[^\W\d]|\$)[\w\$]*)', bygroups(Operator, Name.Attribute)), + (r'([^\W\d]|\$)[\w\$]*:', Name.Label), + (r'([^\W\d]|\$)[\w\$]*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -71,10 +71,10 @@ class JavaLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'([^\W\d]|\$)[\w\$]*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } @@ -340,14 +340,14 @@ class GosuLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # modifiers etc. - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # modifiers etc. + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(in|as|typeof|statictypeof|typeis|typeas|if|else|foreach|for|' r'index|while|do|continue|break|return|try|catch|finally|this|' r'throw|new|switch|case|default|eval|super|outer|classpath|' @@ -360,16 +360,16 @@ class GosuLexer(RegexLexer): Keyword.Type), (r'(package)(\s+)', bygroups(Keyword.Namespace, Text)), (r'(true|false|null|NaN|Infinity)\b', Keyword.Constant), - (r'(class|interface|enhancement|enum)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(class|interface|enhancement|enum)(\s+)([a-zA-Z_]\w*)', bygroups(Keyword.Declaration, Text, Name.Class)), - (r'(uses)(\s+)([a-zA-Z0-9_.]+\*?)', + (r'(uses)(\s+)([\w.]+\*?)', bygroups(Keyword.Namespace, Text, Name.Namespace)), (r'"', String, 'string'), - (r'(\??[\.#])([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\??[\.#])([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'(:)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(:)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_\$]\w*', Name), (r'and|or|not|[\\~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'[0-9]+', Number.Integer), @@ -437,15 +437,20 @@ class GroovyLexer(RegexLexer): tokens = { 'root': [ + # Groovy allows a file to start with a shebang + (r'#!(.*?)$', Comment.Preproc, 'base'), + (r'', Text, 'base'), + ], + 'base': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|goto|instanceof|new|return|switch|this|throw|try|while|in|as)\b', Keyword), @@ -464,9 +469,9 @@ class GroovyLexer(RegexLexer): (r'\$/((?!/\$).)*/\$', String), (r'/(\\\\|\\"|[^/])*/', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'", String.Char), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -474,13 +479,16 @@ class GroovyLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } + def analyse_text(text): + return shebang_matches(text, r'groovy') + class IokeLexer(RegexLexer): """ @@ -550,8 +558,8 @@ class IokeLexer(RegexLexer): (r'#r\[', String.Regex, 'squareRegexp'), #Symbols - (r':[a-zA-Z0-9_!:?]+', String.Symbol), - (r'[a-zA-Z0-9_!:?]+:(?![a-zA-Z0-9_!?])', String.Other), + (r':[\w!:?]+', String.Symbol), + (r'[\w!:?]+:(?![\w!?])', String.Other), (r':"(\\\\|\\"|[^"])*"', String.Symbol), #Documentation @@ -564,10 +572,10 @@ class IokeLexer(RegexLexer): (r'#\[', String, 'squareText'), #Mimic - (r'[a-zA-Z0-9_][a-zA-Z0-9!?_:]+(?=\s*=.*mimic\s)', Name.Entity), + (r'\w[a-zA-Z0-9!?_:]+(?=\s*=.*mimic\s)', Name.Entity), #Assignment - (r'[a-zA-Z_][a-zA-Z0-9_!:?]*(?=[\s]*[+*/-]?=[^=].*($|\.))', + (r'[a-zA-Z_][\w!:?]*(?=[\s]*[+*/-]?=[^=].*($|\.))', Name.Variable), # keywords @@ -658,17 +666,17 @@ class IokeLexer(RegexLexer): r'\+>|!>|<>|&>|%>|#>|\@>|\/>|\*>|\?>|\|>|\^>|~>|\$>|<\->|\->|' r'<<|>>|\*\*|\?\||\?&|\|\||>|<|\*|\/|%|\+|\-|&|\^|\||=|\$|!|~|' u'\\?|#|\u2260|\u2218|\u2208|\u2209)', Operator), - (r'(and|nand|or|xor|nor|return|import)(?![a-zA-Z0-9_!?])', + (r'(and|nand|or|xor|nor|return|import)(?![\w!?])', Operator), # Punctuation (r'(\`\`|\`|\'\'|\'|\.|\,|@@|@|\[|\]|\(|\)|{|})', Punctuation), #kinds - (r'[A-Z][a-zA-Z0-9_!:?]*', Name.Class), + (r'[A-Z][\w!:?]*', Name.Class), #default cellnames - (r'[a-z_][a-zA-Z0-9_!:?]*', Name) + (r'[a-z_][\w!:?]*', Name) ] } @@ -840,14 +848,14 @@ class TeaLangLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(and|break|else|foreach|if|in|not|or|reverse)\b', Keyword), (r'(as|call|define)\b', Keyword.Declaration), @@ -856,9 +864,9 @@ class TeaLangLexer(RegexLexer): (r'(import)(\s+)', bygroups(Keyword.Namespace, Text), 'import'), (r'"(\\\\|\\"|[^"])*"', String), (r'\'(\\\\|\\\'|[^\'])*\'', String), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'(isa|[.]{3}|[.]{2}|[=#!<>+-/%&;,.\*\\\(\)\[\]\{\}])', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -866,10 +874,10 @@ class TeaLangLexer(RegexLexer): (r'\n', Text) ], 'template': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } @@ -894,9 +902,9 @@ class CeylonLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), @@ -919,10 +927,10 @@ class CeylonLexer(RegexLexer): (r'"(\\\\|\\"|[^"])*"', String), (r"'\\.'|'[^\\]'|'\\\{#[0-9a-fA-F]{4}\}'", String.Char), (r'".*``.*``.*"', String.Interpol), - (r'(\.)([a-z_][a-zA-Z0-9_]*)', + (r'(\.)([a-z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'\d{1,3}(_\d{3})+\.\d{1,3}(_\d{3})+[kMGTPmunpf]?', Number.Float), (r'\d{1,3}(_\d{3})+\.[0-9]+([eE][+-]?[0-9]+)?[kMGTPmunpf]?', @@ -939,10 +947,10 @@ class CeylonLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[A-Za-z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[A-Za-z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-z][a-zA-Z0-9_.]*', + (r'[a-z][\w.]*', Name.Namespace, '#pop') ], 'comment': [ @@ -1034,14 +1042,14 @@ class XtendLexer(RegexLexer): tokens = { 'root': [ # method names - (r'^(\s*(?:[a-zA-Z_][a-zA-Z0-9_\.\[\]]*\s+)+?)' # return arguments - r'([a-zA-Z_$][a-zA-Z0-9_$]*)' # method name - r'(\s*)(\()', # signature start + (r'^(\s*(?:[a-zA-Z_][\w\.\[\]]*\s+)+?)' # return arguments + r'([a-zA-Z_$][\w$]*)' # method name + r'(\s*)(\()', # signature start bygroups(using(this), Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(assert|break|case|catch|continue|default|do|else|finally|for|' r'if|goto|instanceof|new|return|switch|this|throw|try|while|IF|' r'ELSE|ELSEIF|ENDIF|FOR|ENDFOR|SEPARATOR|BEFORE|AFTER)\b', @@ -1060,8 +1068,8 @@ class XtendLexer(RegexLexer): (u'(\u00BB)', String, 'template'), (r'"(\\\\|\\"|[^"])*"', String), (r"'(\\\\|\\'|[^'])*'", String), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?-]', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), @@ -1069,10 +1077,10 @@ class XtendLexer(RegexLexer): (r'\n', Text) ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], 'template': [ (r"'''", String, '#pop'), @@ -1112,7 +1120,7 @@ class PigLexer(RegexLexer): (r'0x[0-9a-f]+', Number.Hex), (r'[0-9]+L?', Number.Integer), (r'\n', Text), - (r'([a-z_][a-z0-9_]*)(\s*)(\()', + (r'([a-z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), (r'[()#:]', Text), (r'[^(:#\'\")\s]+', Text), @@ -1175,8 +1183,8 @@ class GoloLexer(RegexLexer): (r'(module|import)(\s+)', bygroups(Keyword.Namespace, Text), 'modname'), - (r'\b([a-zA-Z_][a-z$A-Z0-9._]*)(::)', bygroups(Name.Namespace, Punctuation)), - (r'\b([a-zA-Z_][a-z$A-Z0-9_]*(?:\.[a-zA-Z_][a-z$A-Z0-9_]*)+)\b', Name.Namespace), + (r'\b([a-zA-Z_][\w$.]*)(::)', bygroups(Name.Namespace, Punctuation)), + (r'\b([a-zA-Z_][\w$]*(?:\.[a-zA-Z_][\w$]*)+)\b', Name.Namespace), (r'(let|var)(\s+)', bygroups(Keyword.Declaration, Text), @@ -1201,7 +1209,7 @@ class GoloLexer(RegexLexer): bygroups(Name.Builtin, Punctuation)), (r'(print|println|readln|raise|fun' r'|asInterfaceInstance)\b', Name.Builtin), - (r'(`?[a-zA-Z_][a-z$A-Z0-9_]*)(\()', + (r'(`?[a-zA-Z_][\w$]*)(\()', bygroups(Name.Function, Punctuation)), (r'-?[\d_]*\.[\d_]*([eE][+-]?\d[\d_]*)?F?', Number.Float), @@ -1210,7 +1218,7 @@ class GoloLexer(RegexLexer): (r'-?\d[\d_]*L', Number.Integer.Long), (r'-?\d[\d_]*', Number.Integer), - ('`?[a-zA-Z_][a-z$A-Z0-9_]*', Name), + ('`?[a-zA-Z_][\w$]*', Name), (r'"""', String, combined('stringescape', 'triplestring')), (r'"', String, combined('stringescape', 'doublestring')), @@ -1220,16 +1228,16 @@ class GoloLexer(RegexLexer): ], 'funcname': [ - (r'`?[a-zA-Z_][a-z$A-Z0-9_]*', Name.Function, '#pop'), + (r'`?[a-zA-Z_][\w$]*', Name.Function, '#pop'), ], 'modname': [ - (r'[a-zA-Z_][a-z$A-Z0-9._]*\*?', Name.Namespace, '#pop') + (r'[a-zA-Z_][\w$.]*\*?', Name.Namespace, '#pop') ], 'structname': [ - (r'`?[a-zA-Z0-9_.]+\*?', Name.Class, '#pop') + (r'`?[\w.]+\*?', Name.Class, '#pop') ], 'varname': [ - (r'`?[a-zA-Z_][a-z$A-Z0-9_]*', Name.Variable, '#pop'), + (r'`?[a-zA-Z_][\w$]*', Name.Variable, '#pop'), ], 'string': [ (r'[^\\\'"\n]+', String), @@ -1258,3 +1266,262 @@ class GoloLexer(RegexLexer): (r'(==|<=|<|>=|>|!=)', Operator), ], } + + +class JasminLexer(RegexLexer): + """ + For `Jasmin <http://jasmin.sourceforge.net/>`_ assembly code. + + .. versionadded:: 2.0 + """ + + name = 'Jasmin' + aliases = ['jasmin', 'jasminxt'] + filenames = ['*.j'] + + _whitespace = r' \n\t\r' + _ws = r'(?:[%s]+)' % _whitespace + _separator = r'%s:=' % _whitespace + _break = r'(?=[%s]|$)' % _separator + _name = r'[^%s]+' % _separator + _unqualified_name = r'(?:[^%s.;\[/]+)' % _separator + + tokens = { + 'default': [ + (r'\n', Text, '#pop'), + (r"'", String.Single, ('#pop', 'quote')), + (r'"', String.Double, 'string'), + (r'=', Punctuation), + (r':', Punctuation, 'label'), + (_ws, Text), + (r';.*', Comment.Single), + (r'(\$[-+])?0x-?[\da-fA-F]+%s' % _break, Number.Hex), + (r'(\$[-+]|\+)?-?\d+%s' % _break, Number.Integer), + (r'-?(\d+\.\d*|\.\d+)([eE][-+]?\d+)?[fFdD]?' + r'[\x00-\x08\x0b\x0c\x0e-\x1f]*%s' % _break, Number.Float), + (r'\$%s' % _name, Name.Variable), + + # Directives + (r'\.annotation%s' % _break, Keyword.Reserved, 'annotation'), + (r'(\.attribute|\.bytecode|\.debug|\.deprecated|\.enclosing|' + r'\.interface|\.line|\.signature|\.source|\.stack|\.var|abstract|' + r'annotation|bridge|class|default|enum|field|final|fpstrict|' + r'interface|native|private|protected|public|signature|static|' + r'synchronized|synthetic|transient|varargs|volatile)%s' % _break, + Keyword.Reserved), + (r'\.catch%s' % _break, Keyword.Reserved, 'caught-exception'), + (r'(\.class|\.implements|\.inner|\.super|inner|invisible|' + r'invisibleparam|outer|visible|visibleparam)%s' % _break, + Keyword.Reserved, 'class/convert-dots'), + (r'\.field%s' % _break, Keyword.Reserved, + ('descriptor/convert-dots', 'field')), + (r'(\.end|\.limit|use)%s' % _break, Keyword.Reserved, + 'no-verification'), + (r'\.method%s' % _break, Keyword.Reserved, 'method'), + (r'\.set%s' % _break, Keyword.Reserved, 'var'), + (r'\.throws%s' % _break, Keyword.Reserved, 'exception'), + (r'(from|offset|to|using)%s' % _break, Keyword.Reserved, 'label'), + (r'is%s' % _break, Keyword.Reserved, + ('descriptor/convert-dots', 'var')), + (r'(locals|stack)%s' % _break, Keyword.Reserved, 'verification'), + (r'method%s' % _break, Keyword.Reserved, 'enclosing-method'), + + # Instructions + (r'(aaload|aastore|aconst_null|aload|aload_0|aload_1|aload_2|' + r'aload_3|aload_w|areturn|arraylength|astore|astore_0|astore_1|' + r'astore_2|astore_3|astore_w|athrow|baload|bastore|bipush|' + r'breakpoint|caload|castore|d2f|d2i|d2l|dadd|daload|dastore|' + r'dcmpg|dcmpl|dconst_0|dconst_1|ddiv|dload|dload_0|dload_1|' + r'dload_2|dload_3|dload_w|dmul|dneg|drem|dreturn|dstore|dstore_0|' + r'dstore_1|dstore_2|dstore_3|dstore_w|dsub|dup|dup2|dup2_x1|' + r'dup2_x2|dup_x1|dup_x2|f2d|f2i|f2l|fadd|faload|fastore|fcmpg|' + r'fcmpl|fconst_0|fconst_1|fconst_2|fdiv|fload|fload_0|fload_1|' + r'fload_2|fload_3|fload_w|fmul|fneg|frem|freturn|fstore|fstore_0|' + r'fstore_1|fstore_2|fstore_3|fstore_w|fsub|i2b|i2c|i2d|i2f|i2l|' + r'i2s|iadd|iaload|iand|iastore|iconst_0|iconst_1|iconst_2|' + r'iconst_3|iconst_4|iconst_5|iconst_m1|idiv|iinc|iinc_w|iload|' + r'iload_0|iload_1|iload_2|iload_3|iload_w|imul|ineg|int2byte|' + r'int2char|int2short|ior|irem|ireturn|ishl|ishr|istore|istore_0|' + r'istore_1|istore_2|istore_3|istore_w|isub|iushr|ixor|l2d|l2f|' + r'l2i|ladd|laload|land|lastore|lcmp|lconst_0|lconst_1|ldc2_w|' + r'ldiv|lload|lload_0|lload_1|lload_2|lload_3|lload_w|lmul|lneg|' + r'lookupswitch|lor|lrem|lreturn|lshl|lshr|lstore|lstore_0|' + r'lstore_1|lstore_2|lstore_3|lstore_w|lsub|lushr|lxor|' + r'monitorenter|monitorexit|nop|pop|pop2|ret|ret_w|return|saload|' + r'sastore|sipush|swap)%s' % _break, Keyword.Reserved), + (r'(anewarray|checkcast|instanceof|ldc|ldc_w|new)%s' % _break, + Keyword.Reserved, 'class/no-dots'), + (r'(invokedynamic|invokeinterface|invokenonvirtual|invokespecial|' + r'invokestatic|invokevirtual)%s' % _break, Keyword.Reserved, + 'invocation'), + (r'(getfield|putfield)%s' % _break, Keyword.Reserved, + ('descriptor/no-dots', 'field')), + (r'(getstatic|putstatic)%s' % _break, Keyword.Reserved, + ('descriptor/no-dots', 'static')), + (r'(goto|goto_w|if_acmpeq|if_acmpne|if_icmpeq|if_icmpge|if_icmpgt|' + r'if_icmple|if_icmplt|if_icmpne|ifeq|ifge|ifgt|ifle|iflt|ifne|' + r'ifnonnull|ifnull|jsr|jsr_w)%s' % _break, Keyword.Reserved, + 'label'), + (r'(multianewarray|newarray)%s' % _break, Keyword.Reserved, + 'descriptor/convert-dots'), + (r'tableswitch%s' % _break, Keyword.Reserved, 'table') + ], + 'quote': [ + (r"'", String.Single, '#pop'), + (r'\\u[\da-fA-F]{4}', String.Escape), + (r"[^'\\]+", String.Single) + ], + 'string': [ + (r'"', String.Double, '#pop'), + (r'\\([nrtfb"\'\\]|u[\da-fA-F]{4}|[0-3]?[0-7]{1,2})', + String.Escape), + (r'[^"\\]+', String.Double) + ], + 'root': [ + (r'\n+', Text), + (r"'", String.Single, 'quote'), + include('default'), + (r'(%s)([ \t\r]*)(:)' % _name, + bygroups(Name.Label, Text, Punctuation)), + (_name, String.Other) + ], + 'annotation': [ + (r'\n', Text, ('#pop', 'annotation-body')), + (r'default%s' % _break, Keyword.Reserved, + ('#pop', 'annotation-default')), + include('default') + ], + 'annotation-body': [ + (r'\n+', Text), + (r'\.end%s' % _break, Keyword.Reserved, '#pop'), + include('default'), + (_name, String.Other, ('annotation-items', 'descriptor/no-dots')) + ], + 'annotation-default': [ + (r'\n+', Text), + (r'\.end%s' % _break, Keyword.Reserved, '#pop'), + include('default'), + (r'', Text, ('annotation-items', 'descriptor/no-dots')) + ], + 'annotation-items': [ + (r"'", String.Single, 'quote'), + include('default'), + (_name, String.Other) + ], + 'caught-exception': [ + (r'all%s' % _break, Keyword, '#pop'), + include('exception') + ], + 'class/convert-dots': [ + include('default'), + (r'(L)((?:%s[/.])*)(%s)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'((?:%s[/.])*)(%s)' % (_unqualified_name, _name), + bygroups(Name.Namespace, Name.Class), '#pop') + ], + 'class/no-dots': [ + include('default'), + (r'\[+', Punctuation, ('#pop', 'descriptor/no-dots')), + (r'(L)((?:%s/)*)(%s)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'((?:%s/)*)(%s)' % (_unqualified_name, _name), + bygroups(Name.Namespace, Name.Class), '#pop') + ], + 'descriptor/convert-dots': [ + include('default'), + (r'\[+', Punctuation), + (r'(L)((?:%s[/.])*)(%s?)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'[^%s\[)L]*' % _separator, Keyword.Type, '#pop') + ], + 'descriptor/no-dots': [ + include('default'), + (r'\[+', Punctuation), + (r'(L)((?:%s/)*)(%s)(;)' % (_unqualified_name, _name), + bygroups(Keyword.Type, Name.Namespace, Name.Class, Punctuation), + '#pop'), + (r'[^%s\[)L]*' % _separator, Keyword.Type, '#pop') + ], + 'descriptors/convert-dots': [ + (r'\)', Punctuation, '#pop'), + (r'', Text, 'descriptor/convert-dots') + ], + 'enclosing-method': [ + (_ws, Text), + (r'(?=[^%s]*\()' % _separator, Text, ('#pop', 'invocation')), + (r'', Text, ('#pop', 'class/convert-dots')) + ], + 'exception': [ + include('default'), + (r'((?:%s[/.])*)(%s)' % (_unqualified_name, _name), + bygroups(Name.Namespace, Name.Exception), '#pop') + ], + 'field': [ + (r'static%s' % _break, Keyword.Reserved, ('#pop', 'static')), + include('default'), + (r'((?:%s[/.](?=[^%s]*[/.]))*)(%s[/.])?(%s)' % + (_unqualified_name, _separator, _unqualified_name, _name), + bygroups(Name.Namespace, Name.Class, Name.Variable.Instance), + '#pop') + ], + 'invocation': [ + include('default'), + (r'((?:%s[/.](?=[^%s(]*[/.]))*)(%s[/.])?(%s)(\()' % + (_unqualified_name, _separator, _unqualified_name, _name), + bygroups(Name.Namespace, Name.Class, Name.Function, Punctuation), + ('#pop', 'descriptor/convert-dots', 'descriptors/convert-dots', + 'descriptor/convert-dots')) + ], + 'label': [ + include('default'), + (_name, Name.Label, '#pop') + ], + 'method': [ + include('default'), + (r'(%s)(\()' % _name, bygroups(Name.Function, Punctuation), + ('#pop', 'descriptor/convert-dots', 'descriptors/convert-dots', + 'descriptor/convert-dots')) + ], + 'no-verification': [ + (r'(locals|method|stack)%s' % _break, Keyword.Reserved, '#pop'), + include('default') + ], + 'static': [ + include('default'), + (r'((?:%s[/.](?=[^%s]*[/.]))*)(%s[/.])?(%s)' % + (_unqualified_name, _separator, _unqualified_name, _name), + bygroups(Name.Namespace, Name.Class, Name.Variable.Class), '#pop') + ], + 'table': [ + (r'\n+', Text), + (r'default%s' % _break, Keyword.Reserved, '#pop'), + include('default'), + (_name, Name.Label) + ], + 'var': [ + include('default'), + (_name, Name.Variable, '#pop') + ], + 'verification': [ + include('default'), + (r'(Double|Float|Integer|Long|Null|Top|UninitializedThis)%s' % + _break, Keyword, '#pop'), + (r'Object%s' % _break, Keyword, ('#pop', 'class/no-dots')), + (r'Uninitialized%s' % _break, Keyword, ('#pop', 'label')) + ] + } + + def analyse_text(text): + score = 0 + if re.search(r'^\s*\.class\s', text, re.MULTILINE): + score += 0.5 + if re.search(r'^\s*[a-z]+_[a-z]+\b', text, re.MULTILINE): + score += 0.3 + if re.search(r'^\s*\.(attribute|bytecode|debug|deprecated|enclosing|' + r'inner|interface|limit|set|signature|stack)\b', text, + re.MULTILINE): + score += 0.6 + return score diff --git a/pygments/lexers/math.py b/pygments/lexers/math.py index cfbfc432..de766bd2 100644 --- a/pygments/lexers/math.py +++ b/pygments/lexers/math.py @@ -97,8 +97,8 @@ class JuliaLexer(RegexLexer): (r'[E]?"', String, combined('stringescape', 'string')), # names - (r'@[a-zA-Z0-9_.]+', Name.Decorator), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'@[\w.]+', Name.Decorator), + (r'[a-zA-Z_]\w*', Name), # numbers (r'(\d+(_\d+)+\.\d*|\d*\.\d+(_\d+)+)([eEf][+-]?[0-9]+)?', Number.Float), @@ -116,13 +116,13 @@ class JuliaLexer(RegexLexer): ], 'funcname': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Function, '#pop'), + ('[a-zA-Z_]\w*', Name.Function, '#pop'), ('\([^\s\w{]{1,2}\)', Operator, '#pop'), ('[^\s\w{]{1,2}', Operator, '#pop'), ], 'typename': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + ('[a-zA-Z_]\w*', Name.Class, '#pop') ], 'stringescape': [ @@ -133,7 +133,7 @@ class JuliaLexer(RegexLexer): 'string': [ (r'"', String, '#pop'), (r'\\\\|\\"|\\\n', String.Escape), # included here for raw strings - (r'\$(\([a-zA-Z0-9_]+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?', + (r'\$(\(\w+\))?[-#0 +]*([0-9]+|[*])?(\.([0-9]+|[*]))?', String.Interpol), (r'[^\\"$]+', String), # quotes, dollar signs, and backslashes must be parsed one at a time @@ -357,7 +357,7 @@ class MatlabLexer(RegexLexer): (r'\d+', Number.Integer), (r'(?<![\w\)\].])\'', String, 'string'), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), (r'.', Text), ], 'string': [ @@ -816,7 +816,7 @@ class OctaveLexer(RegexLexer): (r'(?<=[\w\)\].])\'+', Operator), (r'(?<![\w\)\].])\'', String, 'string'), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), (r'.', Text), ], 'string': [ @@ -881,7 +881,7 @@ class ScilabLexer(RegexLexer): (r'\d+[eEf][+-]?[0-9]+', Number.Float), (r'\d+', Number.Integer), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), (r'.', Text), ], 'string': [ @@ -1461,7 +1461,7 @@ class BugsLexer(RegexLexer): % r'|'.join(_FUNCTIONS + _DISTRIBUTIONS), Name.Builtin), # Regular variable names - (r'[A-Za-z][A-Za-z0-9_.]*', Name), + (r'[A-Za-z][\w.]*', Name), # Number Literals (r'[-+]?[0-9]*\.?[0-9]+([eE][-+]?[0-9]+)?', Number), # Punctuation @@ -1521,7 +1521,7 @@ class JagsLexer(RegexLexer): ], 'names' : [ # Regular variable names - (r'[a-zA-Z][a-zA-Z0-9_.]*\b', Name), + (r'[a-zA-Z][\w.]*\b', Name), ], 'comments' : [ # do not use stateful comments @@ -1572,9 +1572,8 @@ class JagsLexer(RegexLexer): class StanLexer(RegexLexer): """Pygments Lexer for Stan models. - The Stan modeling language is specified in the *Stan 2.0.1 - Modeling Language Manual* `pdf - <https://github.com/stan-dev/stan/releases/download/v2.0.1/stan-reference-2.0.1.pdf>`__ + The Stan modeling language is specified in the *Stan Modeling Language User's Guide and Reference Manual, v2.2.0*, + `pdf <https://github.com/stan-dev/stan/releases/download/v2.2.0/stan-reference-2.2.0.pdf>`__. .. versionadded:: 1.6 """ @@ -1619,10 +1618,10 @@ class StanLexer(RegexLexer): + _stan_builtins.DISTRIBUTIONS), Name.Builtin), # Special names ending in __, like lp__ - (r'[A-Za-z][A-Za-z0-9_]*__\b', Name.Builtin.Pseudo), + (r'[A-Za-z]\w*__\b', Name.Builtin.Pseudo), (r'(%s)\b' % r'|'.join(_stan_builtins.RESERVED), Keyword.Reserved), # Regular variable names - (r'[A-Za-z][A-Za-z0-9_]*\b', Name), + (r'[A-Za-z]\w*\b', Name), # Real Literals (r'-?[0-9]+(\.[0-9]+)?[eE]-?[0-9]+', Number.Float), (r'-?[0-9]*\.[0-9]*', Number.Float), @@ -1656,6 +1655,8 @@ class IDLLexer(RegexLexer): filenames = ['*.pro'] mimetypes = ['text/idl'] + flags = re.IGNORECASE | re.MULTILINE + _RESERVED = ['and', 'begin', 'break', 'case', 'common', 'compile_opt', 'continue', 'do', 'else', 'end', 'endcase', 'elseelse', 'endfor', 'endforeach', 'endif', 'endrep', 'endswitch', @@ -1677,7 +1678,7 @@ class IDLLexer(RegexLexer): 'broyden', 'butterworth', 'bytarr', 'byte', 'byteorder', 'bytscl', 'caldat', 'calendar', 'call_external', 'call_function', 'call_method', 'call_procedure', 'canny', - 'catch', 'cd', 'cdf_[0-9a-za-z_]*', 'ceil', 'chebyshev', + 'catch', 'cd', 'cdf_\w*', 'ceil', 'chebyshev', 'check_math', 'chisqr_cvf', 'chisqr_pdf', 'choldc', 'cholsol', 'cindgen', 'cir_3pnt', 'close', 'cluster', 'cluster_tree', 'clust_wts', @@ -1711,7 +1712,7 @@ class IDLLexer(RegexLexer): 'dlm_load', 'dlm_register', 'doc_library', 'double', 'draw_roi', 'edge_dog', 'efont', 'eigenql', 'eigenvec', 'ellipse', 'elmhes', 'emboss', 'empty', 'enable_sysrtn', - 'eof', 'eos_[0-9a-za-z_]*', 'erase', 'erf', 'erfc', 'erfcx', + 'eof', 'eos_\w*', 'erase', 'erf', 'erfc', 'erfcx', 'erode', 'errorplot', 'errplot', 'estimator_filter', 'execute', 'exit', 'exp', 'expand', 'expand_path', 'expint', 'extrac', 'extract_slice', 'factorial', 'fft', 'filepath', @@ -1728,11 +1729,11 @@ class IDLLexer(RegexLexer): 'gauss_cvf', 'gauss_pdf', 'gauss_smooth', 'getenv', 'getwindows', 'get_drive_list', 'get_dxf_objects', 'get_kbrd', 'get_login_info', 'get_lun', 'get_screen_size', - 'greg2jul', 'grib_[0-9a-za-z_]*', 'grid3', 'griddata', + 'greg2jul', 'grib_\w*', 'grid3', 'griddata', 'grid_input', 'grid_tps', 'gs_iter', - 'h5[adfgirst]_[0-9a-za-z_]*', 'h5_browser', 'h5_close', + 'h5[adfgirst]_\w*', 'h5_browser', 'h5_close', 'h5_create', 'h5_get_libversion', 'h5_open', 'h5_parse', - 'hanning', 'hash', 'hdf_[0-9a-za-z_]*', 'heap_free', + 'hanning', 'hash', 'hdf_\w*', 'heap_free', 'heap_gc', 'heap_nosave', 'heap_refcount', 'heap_save', 'help', 'hilbert', 'histogram', 'hist_2d', 'hist_equal', 'hls', 'hough', 'hqr', 'hsv', 'h_eq_ct', 'h_eq_int', @@ -1780,7 +1781,7 @@ class IDLLexer(RegexLexer): 'modifyct', 'moment', 'morph_close', 'morph_distance', 'morph_gradient', 'morph_hitormiss', 'morph_open', 'morph_thin', 'morph_tophat', 'multi', 'm_correlate', - 'ncdf_[0-9a-za-z_]*', 'newton', 'noise_hurl', 'noise_pick', + 'ncdf_\w*', 'newton', 'noise_hurl', 'noise_pick', 'noise_scatter', 'noise_slur', 'norm', 'n_elements', 'n_params', 'n_tags', 'objarr', 'obj_class', 'obj_destroy', 'obj_hasmethod', 'obj_isa', 'obj_new', 'obj_valid', @@ -2182,7 +2183,7 @@ class IgorLexer(RegexLexer): # Compiler directives. (r'^#(include|pragma|define|ifdef|ifndef|endif)', Name.Decorator), - (r'[^a-zA-Z"/]+$', Text), + (r'[^a-z"/]+$', Text), (r'.', Text), ], } diff --git a/pygments/lexers/other.py b/pygments/lexers/other.py index c308c884..d4d1d34b 100644 --- a/pygments/lexers/other.py +++ b/pygments/lexers/other.py @@ -37,7 +37,8 @@ __all__ = ['BrainfuckLexer', 'BefungeLexer', 'RedcodeLexer', 'MOOCodeLexer', 'MscgenLexer', 'KconfigLexer', 'VGLLexer', 'SourcePawnLexer', 'RobotFrameworkLexer', 'PuppetLexer', 'NSISLexer', 'RPMSpecLexer', 'CbmBasicV2Lexer', 'AutoItLexer', 'RexxLexer', 'APLLexer', - 'LSLLexer', 'AmbientTalkLexer', 'PawnLexer', 'VCTreeStatusLexer'] + 'LSLLexer', 'AmbientTalkLexer', 'PawnLexer', 'VCTreeStatusLexer', + 'RslLexer', 'PanLexer', 'RedLexer'] class LSLLexer(RegexLexer): @@ -94,7 +95,7 @@ class LSLLexer(RegexLexer): (lsl_invalid_unimplemented, Error), (lsl_reserved_godmode, Keyword.Reserved), (lsl_reserved_log, Keyword.Reserved), - (r'\b([a-zA-Z_][a-zA-Z0-9_]*)\b', Name.Variable), + (r'\b([a-zA-Z_]\w*)\b', Name.Variable), (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d*', Number.Float), (r'(\d+\.\d*|\.\d+)', Number.Float), (r'0[xX][0-9a-fA-F]+', Number.Hex), @@ -155,15 +156,15 @@ class ECLLexer(RegexLexer): include('hash'), (r'"', String, 'string'), (r'\'', String, 'string'), - (r'(\d+\.\d*|\.\d+|\d+)[eE][+-]?\d+[LlUu]*', Number.Float), + (r'(\d+\.\d*|\.\d+|\d+)e[+-]?\d+[lu]*', Number.Float), (r'(\d+\.\d*|\.\d+|\d+[fF])[fF]?', Number.Float), - (r'0x[0-9a-fA-F]+[LlUu]*', Number.Hex), - (r'0[0-7]+[LlUu]*', Number.Oct), + (r'0x[0-9a-f]+[lu]*', Number.Hex), + (r'0[0-7]+[lu]*', Number.Oct), (r'\d+[LlUu]*', Number.Integer), (r'\*/', Error), (r'[~!%^&*+=|?:<>/-]+', Operator), (r'[{}()\[\],.;]', Punctuation), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), ], 'hash': [ (r'^#.*$', Comment.Preproc), @@ -513,7 +514,7 @@ class LogtalkLexer(RegexLexer): (r'0x[0-9a-fA-F]+', Number), (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), # Variables - (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), + (r'([A-Z_]\w*)', Name.Variable), # Event handlers (r'(after|before)(?=[(])', Keyword), # Execution-context methods @@ -629,7 +630,7 @@ class LogtalkLexer(RegexLexer): # Ponctuation (r'[()\[\],.|]', Text), # Atoms - (r"[a-z][a-zA-Z0-9_]*", Text), + (r"[a-z]\w*", Text), (r"'", String, 'quoted_atom'), ], @@ -660,8 +661,8 @@ class LogtalkLexer(RegexLexer): (r'op(?=[(])', Keyword, 'root'), (r'(c(alls|oinductive)|reexport|use(s|_module))(?=[(])', Keyword, 'root'), - (r'[a-z][a-zA-Z0-9_]*(?=[(])', Text, 'root'), - (r'[a-z][a-zA-Z0-9_]*[.]', Text, 'root'), + (r'[a-z]\w*(?=[(])', Text, 'root'), + (r'[a-z]\w*[.]', Text, 'root'), ], 'entityrelations': [ @@ -674,9 +675,9 @@ class LogtalkLexer(RegexLexer): (r'0x[0-9a-fA-F]+', Number), (r'\d+\.?\d*((e|E)(\+|-)?\d+)?', Number), # Variables - (r'([A-Z_][a-zA-Z0-9_]*)', Name.Variable), + (r'([A-Z_]\w*)', Name.Variable), # Atoms - (r"[a-z][a-zA-Z0-9_]*", Text), + (r"[a-z]\w*", Text), (r"'", String, 'quoted_atom'), # Strings (r'"(\\\\|\\"|[^"])*"', String), @@ -746,11 +747,11 @@ class GnuplotLexer(RegexLexer): (_shortened_many('pwd$', 're$read', 'res$et', 'scr$eendump', 'she$ll', 'test$'), Keyword, 'noargs'), - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(=)', + ('([a-zA-Z_]\w*)(\s*)(=)', bygroups(Name.Variable, Text, Operator), 'genericargs'), - ('([a-zA-Z_][a-zA-Z0-9_]*)(\s*\(.*?\)\s*)(=)', + ('([a-zA-Z_]\w*)(\s*\(.*?\)\s*)(=)', bygroups(Name.Function, Text, Operator), 'genericargs'), - (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), # macros + (r'@[a-zA-Z_]\w*', Name.Constant), # macros (r';', Keyword), ], 'comment': [ @@ -796,10 +797,10 @@ class GnuplotLexer(RegexLexer): ('[,.~!%^&*+=|?:<>/-]', Operator), ('[{}()\[\]]', Punctuation), (r'(eq|ne)\b', Operator.Word), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(\()', + (r'([a-zA-Z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), - (r'@[a-zA-Z_][a-zA-Z0-9_]*', Name.Constant), # macros + (r'[a-zA-Z_]\w*', Name), + (r'@[a-zA-Z_]\w*', Name.Constant), # macros (r'\\\n', Text), ], 'optionarg': [ @@ -1305,28 +1306,26 @@ class ModelicaLexer(RegexLexer): (r'\d+[Ll]?', Number.Integer), (r'[~!%^&*+=|?:<>/-]', Operator), (r'(true|false|NULL|Real|Integer|Boolean)\b', Name.Builtin), - (r'([a-zA-Z_][\w]*|[\'][^\']+[\'])' + (r'([a-z_][\w]*|\'[^\']+\')' r'([\[\d,:\]]*)' - r'(\.([a-zA-Z_][\w]*|[\'][^\']+[\']))+' + r'(\.([a-z_][\w]*|\'[^\']+\'))+' r'([\[\d,:\]]*)', Name.Class), - (r'([a-zA-Z_][\w]*|[\'][^\']+[\'])' - r'([\[\d,:\]]+)', Name.Class), (r'(\'[\w\+\-\*\/\^]+\'|\w+)', Name), (r'[()\[\]{},.;]', Punctuation), (r'\'', Name, 'quoted_ident'), ], 'root': [ include('whitespace'), - include('keywords'), include('classes'), include('functions'), + include('keywords'), include('operators'), (r'("<html>|<html>)', Name.Tag, 'html-content'), include('statements'), ], 'keywords': [ (r'(algorithm|annotation|break|connect|constant|constrainedby|' - r'discrete|each|else|elseif|elsewhen|encapsulated|enumeration|' + r'discrete|each|end|else|elseif|elsewhen|encapsulated|enumeration|' r'equation|exit|expandable|extends|' r'external|false|final|flow|for|if|import|impure|in|initial\sequation|' r'inner|input|loop|nondiscrete|outer|output|parameter|partial|' @@ -1335,9 +1334,11 @@ class ModelicaLexer(RegexLexer): ], 'functions': [ (r'(abs|acos|acosh|asin|asinh|atan|atan2|atan3|ceil|cos|cosh|' - r'cross|div|exp|floor|getInstanceName|log|log10|mod|rem|' - r'semiLinear|sign|sin|sinh|size|spatialDistribution|sqrt|tan|' - r'tanh|zeros)\b', Name.Function), + r'cross|diagonal|div|exp|fill|floor|getInstanceName|identity|' + r'linspace|log|log10|matrix|mod|max|min|ndims|ones|outerProduct|' + r'product|rem|scalar|semiLinear|skew|sign|sin|sinh|size|' + r'spatialDistribution|sum|sqrt|symmetric|tan|tanh|transpose|' + r'vector|zeros)\b', Name.Function), ], 'operators': [ (r'(actualStream|and|assert|backSample|cardinality|change|Clock|' @@ -1347,8 +1348,9 @@ class ModelicaLexer(RegexLexer): r'terminate)\b', Name.Builtin), ], 'classes': [ - (r'(operator)?(\s+)?(block|class|connector|end|function|model|operator|package|' - r'record|type)(\s+)((?!if|when|while)[A-Za-z_]\w*|[\'][^\']+[\'])([;]?)', + (r'(operator)?(\s+)?(block|class|connector|end|function|model|' + r'operator|package|record|type)(\s+)' + r'((?!if|for|when|while)[a-z_]\w*|\'[^\']+\')([;]?)', bygroups(Keyword, Text, Keyword, Text, Name.Class, Text)) ], 'quoted_ident': [ @@ -1357,7 +1359,7 @@ class ModelicaLexer(RegexLexer): ], 'string': [ (r'"', String, '#pop'), - (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|[0-7]{1,3})', + (r'\\([\\abfnrtv"\']|x[a-f0-9]{2,4}|[0-7]{1,3})', String.Escape), (r'[^\\"\n]+', String), # all other characters (r'\\\n', String), # line continuation @@ -1378,14 +1380,14 @@ class RebolLexer(RegexLexer): """ name = 'REBOL' aliases = ['rebol'] - filenames = ['*.r', '*.r3'] + filenames = ['*.r', '*.r3', '*.reb'] mimetypes = ['text/x-rebol'] flags = re.IGNORECASE | re.MULTILINE re.IGNORECASE - escape_re = r'(?:\^\([0-9a-fA-F]{1,4}\)*)' + escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)' def word_callback(lexer, match): word = match.group() @@ -1481,9 +1483,9 @@ class RebolLexer(RegexLexer): 'script': [ (r'\s+', Text), (r'#"', String.Char, 'char'), - (r'#{[0-9a-fA-F]*}', Number.Hex), + (r'#{[0-9a-f]*}', Number.Hex), (r'2#{', Number.Hex, 'bin2'), - (r'64#{[0-9a-zA-Z+/=\s]*}', Number.Hex), + (r'64#{[0-9a-z+/=\s]*}', Number.Hex), (r'"', String, 'string'), (r'{', String, 'string2'), (r';#+.*\n', Comment.Special), @@ -1491,9 +1493,9 @@ class RebolLexer(RegexLexer): (r';.*\n', Comment), (r'%"', Name.Decorator, 'stringFile'), (r'%[^(\^{^")\s\[\]]+', Name.Decorator), - (r'[+-]?([a-zA-Z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money + (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time - (r'\d+[\-\/][0-9a-zA-Z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?' + (r'\d+[\-\/][0-9a-z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?' r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple (r'\d+[xX]\d+', Keyword.Constant), # pair @@ -1501,13 +1503,16 @@ class RebolLexer(RegexLexer): (r'[+-]?\d+(\'\d+)?[\.,]\d*', Number.Float), (r'[+-]?\d+(\'\d+)?', Number), (r'[\[\]\(\)]', Generic.Strong), - (r'[a-zA-Z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url + (r'[a-z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url (r'mailto:[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # url (r'[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # email - (r'comment\s', Comment, 'comment'), + (r'comment\s"', Comment, 'commentString1'), + (r'comment\s{', Comment, 'commentString2'), + (r'comment\s\[', Comment, 'commentBlock'), + (r'comment\s[^(\s{\"\[]+', Comment), (r'/[^(\^{^")\s/[\]]*', Name.Attribute), (r'([^(\^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback), - (r'<[a-zA-Z0-9:._-]*>', Name.Tag), + (r'<[\w:.-]*>', Name.Tag), (r'<[^(<>\s")]+', Name.Tag, 'tag'), (r'([^(\^{^")\s]+)', Text), ], @@ -1559,12 +1564,6 @@ class RebolLexer(RegexLexer): (r'([0-1]\s*){8}', Number.Hex), (r'}', Number.Hex, '#pop'), ], - 'comment': [ - (r'"', Comment, 'commentString1'), - (r'{', Comment, 'commentString2'), - (r'\[', Comment, 'commentBlock'), - (r'[^(\s{\"\[]+', Comment, '#pop'), - ], 'commentString1': [ (r'[^(\^")]+', Comment), (escape_re, Comment), @@ -1583,7 +1582,9 @@ class RebolLexer(RegexLexer): 'commentBlock': [ (r'\[', Comment, '#push'), (r'\]', Comment, '#pop'), - (r'[^(\[\])]+', Comment), + (r'"', Comment, "commentString1"), + (r'{', Comment, "commentString2"), + (r'[^(\[\]\"{)]+', Comment), ], } def analyse_text(text): @@ -1618,7 +1619,7 @@ class ABAPLexer(RegexLexer): (r'\".*?\n', Comment.Single), ], 'variable-names': [ - (r'<[\S_]+>', Name.Variable), + (r'<\S+>', Name.Variable), (r'\w[\w~]*(?:(\[\])|->\*)?', Name.Variable), ], 'root': [ @@ -1802,15 +1803,15 @@ class NewspeakLexer(RegexLexer): 'root' : [ (r'\b(Newsqueak2)\b',Keyword.Declaration), (r"'[^']*'",String), - (r'\b(class)(\s+)([a-zA-Z0-9_]+)(\s*)', + (r'\b(class)(\s+)(\w+)(\s*)', bygroups(Keyword.Declaration,Text,Name.Class,Text)), (r'\b(mixin|self|super|private|public|protected|nil|true|false)\b', Keyword), - (r'([a-zA-Z0-9_]+\:)(\s*)([a-zA-Z_]\w+)', + (r'(\w+\:)(\s*)([a-zA-Z_]\w+)', bygroups(Name.Function,Text,Name.Variable)), - (r'([a-zA-Z0-9_]+)(\s*)(=)', + (r'(\w+)(\s*)(=)', bygroups(Name.Attribute,Text,Operator)), - (r'<[a-zA-Z0-9_]+>', Comment.Special), + (r'<\w+>', Comment.Special), include('expressionstat'), include('whitespace') ], @@ -1864,7 +1865,7 @@ class GherkinLexer(RegexLexer): tokens = { 'comments': [ - (r'#.*$', Comment), + (r'^\s*#.*$', Comment), ], 'feature_elements' : [ (step_keywords, Keyword, "step_content_stack"), @@ -1889,10 +1890,13 @@ class GherkinLexer(RegexLexer): (r"[^\|]", Name.Variable), ], 'scenario_sections_on_stack': [ - (feature_element_keywords, bygroups(Name.Function, Keyword, Keyword, Name.Function), "feature_elements_on_stack"), + (feature_element_keywords, + bygroups(Name.Function, Keyword, Keyword, Name.Function), + "feature_elements_on_stack"), ], 'narrative': [ include('scenario_sections_on_stack'), + include('comments'), (r"(\s|.)", Name.Function), ], 'table_vars': [ @@ -2018,23 +2022,23 @@ class AsymptoteLexer(RegexLexer): r'bounds|coord|frame|guide|horner|int|linefit|marginT|pair|pen|' r'picture|position|real|revolution|slice|splitface|ticksgridT|' r'tickvalues|tree|triple|vertex|void)\b', Keyword.Type), - ('[a-zA-Z_][a-zA-Z0-9_]*:(?!:)', Name.Label), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*:(?!:)', Name.Label), + ('[a-zA-Z_]\w*', Name), ], 'root': [ include('whitespace'), # functions - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|\*))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|\*))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')({)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation), 'function'), # function declarations - (r'((?:[a-zA-Z0-9_*\s])+?(?:\s|\*))' # return arguments - r'([a-zA-Z_][a-zA-Z0-9_]*)' # method name - r'(\s*\([^;]*?\))' # signature + (r'((?:[\w*\s])+?(?:\s|\*))' # return arguments + r'([a-zA-Z_]\w*)' # method name + r'(\s*\([^;]*?\))' # signature r'(' + _ws + r')(;)', bygroups(using(this), Name.Function, using(this), using(this), Punctuation)), @@ -2187,7 +2191,7 @@ class AutohotkeyLexer(RegexLexer): (r'^;.*?$', Comment.Singleline), (r'[]{}(),;[]', Punctuation), (r'(in|is|and|or|not)\b', Operator.Word), - (r'\%[a-zA-Z_#@$][a-zA-Z0-9_#@$]*\%', Name.Variable), + (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable), (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator), include('commands'), include('labels'), @@ -2195,7 +2199,7 @@ class AutohotkeyLexer(RegexLexer): include('builtInVariables'), (r'"', String, combined('stringescape', 'dqs')), include('numbers'), - (r'[a-zA-Z_#@$][a-zA-Z0-9_#@$]*', Name), + (r'[a-zA-Z_#@$][\w#@$]*', Name), (r'\\|\'', Text), (r'\`([\,\%\`abfnrtv\-\+;])', String.Escape), include('garbage'), @@ -2389,7 +2393,7 @@ class MaqlLexer(RegexLexer): r'SYNCHRONIZE|TYPE|DEFAULT|ORDER|ASC|DESC|HYPERLINK|' r'INCLUDE|TEMPLATE|MODIFY)\b', Keyword), # FUNCNAME - (r'[a-zA-Z]\w*\b', Name.Function), + (r'[a-z]\w*\b', Name.Function), # Comments (r'#.*', Comment.Single), # Punctuation @@ -2424,7 +2428,7 @@ class GoodDataCLLexer(RegexLexer): # Comments (r'#.*', Comment.Single), # Function call - (r'[a-zA-Z]\w*', Name.Function), + (r'[a-z]\w*', Name.Function), # Argument list (r'\(', Token.Punctuation, 'args-list'), # Punctuation @@ -2435,7 +2439,7 @@ class GoodDataCLLexer(RegexLexer): 'args-list': [ (r'\)', Token.Punctuation, '#pop'), (r',', Token.Punctuation), - (r'[a-zA-Z]\w*', Name.Variable), + (r'[a-z]\w*', Name.Variable), (r'=', Operator), (r'"', Literal.String, 'string-literal'), (r'[0-9]+(?:\.[0-9]+)?(?:[eE][+-]?[0-9]{1,3})?', Literal.Number), @@ -2487,18 +2491,18 @@ class ProtoBufLexer(RegexLexer): (r'0[0-7]+[LlUu]*', Number.Oct), (r'\d+[LlUu]*', Number.Integer), (r'[+-=]', Operator), - (r'([a-zA-Z_][a-zA-Z0-9_\.]*)([ \t]*)(=)', + (r'([a-zA-Z_][\w\.]*)([ \t]*)(=)', bygroups(Name.Attribute, Text, Operator)), - ('[a-zA-Z_][a-zA-Z0-9_\.]*', Name), + ('[a-zA-Z_][\w\.]*', Name), ], 'package': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Namespace, '#pop') + (r'[a-zA-Z_]\w*', Name.Namespace, '#pop') ], 'message': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'type': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name, '#pop') + (r'[a-zA-Z_]\w*', Name, '#pop') ], } @@ -2521,12 +2525,12 @@ class HybrisLexer(RegexLexer): 'root': [ # method names (r'^(\s*(?:function|method|operator\s+)+?)' - r'([a-zA-Z_][a-zA-Z0-9_]*)' + r'([a-zA-Z_]\w*)' r'(\s*)(\()', bygroups(Keyword, Name.Function, Text, Operator)), (r'[^\S\n]+', Text), (r'//.*?\n', Comment.Single), (r'/\*.*?\*/', Comment.Multiline), - (r'@[a-zA-Z_][a-zA-Z0-9_\.]*', Name.Decorator), + (r'@[a-zA-Z_][\w\.]*', Name.Decorator), (r'(break|case|catch|next|default|do|else|finally|for|foreach|of|' r'unless|if|new|return|switch|me|throw|try|while)\b', Keyword), (r'(extends|private|protected|public|static|throws|function|method|' @@ -2562,10 +2566,10 @@ class HybrisLexer(RegexLexer): r'Exception)\b', Keyword.Type), (r'"(\\\\|\\"|[^"])*"', String), (r"'\\.'|'[^\\]'|'\\u[0-9a-f]{4}'", String.Char), - (r'(\.)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\.)([a-zA-Z_]\w*)', bygroups(Operator, Name.Attribute)), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_\$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_]\w*:', Name.Label), + (r'[a-zA-Z_\$]\w*', Name), (r'[~\^\*!%&\[\]\(\)\{\}<>\|+=:;,./?\-@]+', Operator), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-f]+', Number.Hex), @@ -2573,10 +2577,10 @@ class HybrisLexer(RegexLexer): (r'\n', Text), ], 'class': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_]\w*', Name.Class, '#pop') ], 'import': [ - (r'[a-zA-Z0-9_.]+\*?', Name.Namespace, '#pop') + (r'[\w.]+\*?', Name.Namespace, '#pop') ], } @@ -2625,7 +2629,7 @@ class AwkLexer(RegexLexer): (r'(ARGC|ARGIND|ARGV|CONVFMT|ENVIRON|ERRNO|FIELDWIDTHS|FILENAME|FNR|FS|' r'IGNORECASE|NF|NR|OFMT|OFS|ORFS|RLENGTH|RS|RSTART|RT|' r'SUBSEP)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -2842,7 +2846,7 @@ class UrbiscriptLexer(ExtendedRegexLexer): Operator.Word), (r'[{}\[\]()]+', Punctuation), (r'(?:;|\||,|&|\?|!)+', Punctuation), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'0x[0-9a-fA-F]+', Number.Hex), # Float, Integer, Angle and Duration (r'(?:[0-9]+(?:(?:\.[0-9]+)?(?:[eE][+-]?[0-9]+)?)?' @@ -3048,7 +3052,7 @@ class MscgenLexer(RegexLexer): aliases = ['mscgen', 'msc'] filenames = ['*.msc'] - _var = r'([a-zA-Z0-9_]+|"(?:\\"|[^"])*")' + _var = r'(\w+|"(?:\\"|[^"])*")' tokens = { 'root': [ @@ -3198,9 +3202,9 @@ class VGLLexer(RegexLexer): (r'(true|false|null|empty|error|locked)', Keyword.Constant), (r'[~\^\*\#!%&\[\]\(\)<>\|+=:;,./?-]', Operator), (r'"[^"]*"', String), - (r'(\.)([a-z_\$][a-z0-9_\$]*)', bygroups(Operator, Name.Attribute)), + (r'(\.)([a-z_\$][\w\$]*)', bygroups(Operator, Name.Attribute)), (r'[0-9][0-9]*(\.[0-9]+(e[+\-]?[0-9]+)?)?', Number), - (r'[a-z_\$][a-z0-9_\$]*', Name), + (r'[a-z_\$][\w\$]*', Name), (r'[\r\n]+', Text), (r'\s+', Text) ] @@ -3250,7 +3254,7 @@ class SourcePawnLexer(RegexLexer): r'public|return|sizeof|static|decl|struct|switch)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -3350,7 +3354,7 @@ class PuppetLexer(RegexLexer): ], 'names': [ - ('[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute), + ('[a-zA-Z_]\w*', Name.Attribute), (r'(\$\S+)(\[)(\S+)(\])', bygroups(Name.Variable, Punctuation, String, Punctuation)), (r'\$\S+', Name.Variable), @@ -3391,7 +3395,7 @@ class PuppetLexer(RegexLexer): 'strings': [ (r'"([^"])*"', String), - (r'\'([^\'])*\'', String), + (r"'(\\'|[^'])*'", String), ], } @@ -3587,9 +3591,9 @@ class RPMSpecLexer(RegexLexer): 'interpol': [ (r'%\{?__[a-z_]+\}?', Name.Function), (r'%\{?_([a-z_]+dir|[a-z_]+path|prefix)\}?', Keyword.Pseudo), - (r'%\{\?[A-Za-z0-9_]+\}', Name.Variable), + (r'%\{\?\w+\}', Name.Variable), (r'\$\{?RPM_[A-Z0-9_]+\}?', Name.Variable.Global), - (r'%\{[a-zA-Z][a-zA-Z0-9_]+\}', Keyword.Constant), + (r'%\{[a-zA-Z]\w+\}', Keyword.Constant), ] } @@ -3720,7 +3724,7 @@ class AutoItLexer(RegexLexer): (r'(#comments-start|#cs).*?(#comments-end|#ce)', Comment.Multiline), (r'[\[\]{}(),;]', Punctuation), (r'(and|or|not)\b', Operator.Word), - (r'[\$|@][a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r'[\$|@][a-zA-Z_]\w*', Name.Variable), (r'!=|==|:=|\.=|<<|>>|[-~+/*%=<>&^|?:!.]', Operator), include('commands'), include('labels'), @@ -3728,7 +3732,7 @@ class AutoItLexer(RegexLexer): include('builtInMarcros'), (r'"', String, combined('stringescape', 'dqs')), include('numbers'), - (r'[a-zA-Z_#@$][a-zA-Z0-9_#@$]*', Name), + (r'[a-zA-Z_#@$][\w#@$]*', Name), (r'\\|\'', Text), (r'\`([\,\%\`abfnrtv\-\+;])', String.Escape), (r'_\n', Text), # Line continuation @@ -3796,15 +3800,15 @@ class RexxLexer(RegexLexer): (r'"', String, 'string_double'), (r"'", String, 'string_single'), (r'[0-9]+(\.[0-9]+)?(e[+-]?[0-9])?', Number), - (r'([a-z_][a-z0-9_]*)(\s*)(:)(\s*)(procedure)\b', + (r'([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b', bygroups(Name.Function, Whitespace, Operator, Whitespace, Keyword.Declaration)), - (r'([a-z_][a-z0-9_]*)(\s*)(:)', + (r'([a-z_]\w*)(\s*)(:)', bygroups(Name.Label, Whitespace, Operator)), include('function'), include('keyword'), include('operator'), - (r'[a-z_][a-z0-9_]*', Text), + (r'[a-z_]\w*', Text), ], 'function': [ (r'(abbrev|abs|address|arg|b2x|bitand|bitor|bitxor|c2d|c2x|' @@ -3852,7 +3856,7 @@ class RexxLexer(RegexLexer): _ADDRESS_PATTERN = _c(r'^\s*address\s+') _DO_WHILE_PATTERN = _c(r'^\s*do\s+while\b') _IF_THEN_DO_PATTERN = _c(r'^\s*if\b.+\bthen\s+do\s*$') - _PROCEDURE_PATTERN = _c(r'^\s*([a-z_][a-z0-9_]*)(\s*)(:)(\s*)(procedure)\b') + _PROCEDURE_PATTERN = _c(r'^\s*([a-z_]\w*)(\s*)(:)(\s*)(procedure)\b') _ELSE_DO_PATTERN = _c(r'\belse\s+do\s*$') _PARSE_ARG_PATTERN = _c(r'^\s*parse\s+(upper\s+)?(arg|value)\b') PATTERNS_AND_WEIGHTS = ( @@ -3997,12 +4001,12 @@ class AmbientTalkLexer(RegexLexer): (r'"(\\\\|\\"|[^"])*"', String), (r'\|', Punctuation, 'arglist'), (r'<:|[\^\*!%&<>+=,./?-]|:=', Operator), - (r"`[a-zA-Z_][a-zA-Z0-9_]*", String.Symbol), - (r"[a-zA-Z_][a-zA-Z0-9_]*:", Name.Function), + (r"`[a-zA-Z_]\w*", String.Symbol), + (r"[a-zA-Z_]\w*:", Name.Function), (r"[\{\}()\[\];`]", Punctuation), (r'(self|super)\b', Name.Variable.Instance), - (r"[a-zA-Z_][a-zA-Z0-9_]*", Name.Variable), - (r"@[a-zA-Z_][a-zA-Z0-9_]*", Name.Class), + (r"[a-zA-Z_]\w*", Name.Variable), + (r"@[a-zA-Z_]\w*", Name.Class), (r"@\[", Name.Class, 'annotations'), include('numbers'), ], @@ -4011,9 +4015,9 @@ class AmbientTalkLexer(RegexLexer): (r'\d+', Number.Integer) ], 'namespace': [ - (r'[a-zA-Z_][a-zA-Z0-9_]*\.', Name.Namespace), - (r'[a-zA-Z_][a-zA-Z0-9_]*:', Name.Function , '#pop'), - (r'[a-zA-Z_][a-zA-Z0-9_]*(?!\.)', Name.Function , '#pop') + (r'[a-zA-Z_]\w*\.', Name.Namespace), + (r'[a-zA-Z_]\w*:', Name.Function , '#pop'), + (r'[a-zA-Z_]\w*(?!\.)', Name.Function , '#pop') ], 'annotations' : [ (r"(.*?)\]", Name.Class, '#pop') @@ -4021,7 +4025,7 @@ class AmbientTalkLexer(RegexLexer): 'arglist' : [ (r'\|', Punctuation, '#pop'), (r'\s*(,)\s*', Punctuation), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Variable), + (r'[a-zA-Z_]\w*', Name.Variable), ], } @@ -4068,7 +4072,7 @@ class PawnLexer(RegexLexer): r'public|return|sizeof|tagof|state|goto)\b', Keyword), (r'(bool|Float)\b', Keyword.Type), (r'(true|false)\b', Keyword.Constant), - ('[a-zA-Z_][a-zA-Z0-9_]*', Name), + ('[a-zA-Z_]\w*', Name), ], 'string': [ (r'"', String, '#pop'), @@ -4117,3 +4121,314 @@ class VCTreeStatusLexer(RegexLexer): (r'.*\n', Text) ] } + + +class RslLexer(RegexLexer): + """ + `RSL <http://en.wikipedia.org/wiki/RAISE>`_ is the formal specification + language used in RAISE (Rigorous Approach to Industrial Software Engineering) + method. + + .. versionadded:: 2.0 + """ + name = 'RSL' + aliases = ['rsl'] + filenames = ['*.rsl'] + mimetypes = ['text/rsl'] + + flags = re.MULTILINE | re.DOTALL + + tokens = { + 'root':[ + (r'\b(Bool|Char|Int|Nat|Real|Text|Unit|abs|all|always|any|as|' + r'axiom|card|case|channel|chaos|class|devt_relation|dom|elems|' + r'else|elif|end|exists|extend|false|for|hd|hide|if|in|is|inds|' + r'initialise|int|inter|isin|len|let|local|ltl_assertion|object|' + r'of|out|post|pre|read|real|rng|scheme|skip|stop|swap|then|' + r'thoery|test_case|tl|transition_system|true|type|union|until|' + r'use|value|variable|while|with|write|~isin|-inflist|-infset|' + r'-list|-set)\b', Keyword), + (r'(variable|value)\b', Keyword.Declaration), + (r'--.*?\n', Comment), + (r'<:.*?:>', Comment), + (r'\{!.*?!\}', Comment), + (r'/\*.*?\*/', Comment), + (r'^[ \t]*([\w]+)[ \t]*:[^:]', Name.Function), + (r'(^[ \t]*)([\w]+)([ \t]*\([\w\s,]*\)[ \t]*)(is|as)', + bygroups(Text, Name.Function, Text, Keyword)), + (r'\b[A-Z]\w*\b',Keyword.Type), + (r'(true|false)\b', Keyword.Constant), + (r'".*"',String), + (r'\'.\'',String.Char), + (r'(><|->|-m->|/\\|<=|<<=|<\.|\|\||\|\^\||-~->|-~m->|\\/|>=|>>|' + r'\.>|\+\+|-\\|<->|=>|:-|~=|\*\*|<<|>>=|\+>|!!|\|=\||#)', + Operator), + (r'[0-9]+\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), + (r'0x[0-9a-f]+', Number.Hex), + (r'[0-9]+', Number.Integer), + (r'.', Text), + ], + } + + def analyse_text(text): + """ + Check for the most common text in the beginning of a RSL file. + """ + if re.search(r'scheme\s*.*?=\s*class\s*type', text, re.I) is not None: + return 1.0 + else: + return 0.01 + + +class PanLexer(RegexLexer): + """ + Lexer for `pan <http://github.com/quattor/pan/>`_ source files. + + Based on tcsh lexer. + + .. versionadded:: 2.0 + """ + + name = 'Pan' + aliases = ['pan'] + filenames = ['*.pan'] + + tokens = { + 'root': [ + include('basic'), + (r'\(', Keyword, 'paren'), + (r'{', Keyword, 'curly'), + include('data'), + ], + 'basic': [ + (r'\b(if|for|with|else|type|bind|while|valid|final|prefix|unique|' + r'object|foreach|include|template|function|variable|structure|' + r'extensible|declaration)\s*\b', + Keyword), + (r'\b(file_contents|format|index|length|match|matches|replace|' + r'splice|split|substr|to_lowercase|to_uppercase|debug|error|' + r'traceback|deprecated|base64_decode|base64_encode|digest|escape|' + r'unescape|append|create|first|nlist|key|length|list|merge|next|' + r'prepend|splice|is_boolean|is_defined|is_double|is_list|is_long|' + r'is_nlist|is_null|is_number|is_property|is_resource|is_string|' + r'to_boolean|to_double|to_long|to_string|clone|delete|exists|' + r'path_exists|if_exists|return|value)\s*\b', + Name.Builtin), + (r'#.*', Comment), + (r'\\[\w\W]', String.Escape), + (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)), + (r'[\[\]{}()=]+', Operator), + (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String), + (r';', Punctuation), + ], + 'data': [ + (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double), + (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), + (r'\s+', Text), + (r'[^=\s\[\]{}()$"\'`\\;#]+', Text), + (r'\d+(?= |\Z)', Number), + ], + 'curly': [ + (r'}', Keyword, '#pop'), + (r':-', Keyword), + (r'\w+', Name.Variable), + (r'[^}:"\'`$]+', Punctuation), + (r':', Punctuation), + include('root'), + ], + 'paren': [ + (r'\)', Keyword, '#pop'), + include('root'), + ], + } + +class RedLexer(RegexLexer): + """ + A `Red-language <http://www.red-lang.org/>`_ lexer. + + .. versionadded:: 2.0 + """ + name = 'Red' + aliases = ['red', 'red/system'] + filenames = ['*.red', '*.reds'] + mimetypes = ['text/x-red', 'text/x-red-system'] + + flags = re.IGNORECASE | re.MULTILINE + + escape_re = r'(?:\^\([0-9a-f]{1,4}\)*)' + + def word_callback(lexer, match): + word = match.group() + + if re.match(".*:$", word): + yield match.start(), Generic.Subheading, word + elif re.match( + r'(if|unless|either|any|all|while|until|loop|repeat|' + r'foreach|forall|func|function|does|has|switch|' + r'case|reduce|compose|get|set|print|prin|equal\?|' + r'not-equal\?|strict-equal\?|lesser\?|greater\?|lesser-or-equal\?|' + r'greater-or-equal\?|same\?|not|type\?|stats|' + r'bind|union|replace|charset|routine)$', word): + yield match.start(), Name.Builtin, word + elif re.match( + r'(make|random|reflect|to|form|mold|absolute|add|divide|multiply|negate|' + r'power|remainder|round|subtract|even\?|odd\?|and~|complement|or~|xor~|' + r'append|at|back|change|clear|copy|find|head|head\?|index\?|insert|' + r'length\?|next|pick|poke|remove|reverse|select|sort|skip|swap|tail|tail\?|' + r'take|trim|create|close|delete|modify|open|open\?|query|read|rename|' + r'update|write)$', word): + yield match.start(), Name.Function, word + elif re.match( + r'(yes|on|no|off|true|false|tab|cr|lf|newline|escape|slash|sp|space|null|' + r'none|crlf|dot|null-byte)$', word): + yield match.start(), Name.Builtin.Pseudo, word + elif re.match( + r'(#system-global|#include|#enum|#define|#either|#if|#import|#export|' + r'#switch|#default|#get-definition)$', word): + yield match.start(), Keyword.Namespace, word + elif re.match( + r'(system|halt|quit|quit-return|do|load|q|recycle|call|run|ask|parse|' + r'raise-error|return|exit|break|alias|push|pop|probe|\?\?|spec-of|body-of|' + r'quote|forever)$', word): + yield match.start(), Name.Exception, word + elif re.match( + r'(action\?|block\?|char\?|datatype\?|file\?|function\?|get-path\?|zero\?|' + r'get-word\?|integer\?|issue\?|lit-path\?|lit-word\?|logic\?|native\?|' + r'op\?|paren\?|path\?|refinement\?|set-path\?|set-word\?|string\?|unset\?|' + r'any-struct\?|none\?|word\?|any-series\?)$', word): + yield match.start(), Keyword, word + elif re.match(r'(JNICALL|stdcall|cdecl|infix)$', word): + yield match.start(), Keyword.Namespace, word + elif re.match("to-.*", word): + yield match.start(), Keyword, word + elif re.match('(\+|-|\*|/|//|\*\*|and|or|xor|=\?|=|==|===|<>|<|>|<=|>=|<<|>>|<<<|>>>|%|-\*\*)$', word): + yield match.start(), Operator, word + elif re.match(".*\!$", word): + yield match.start(), Keyword.Type, word + elif re.match("'.*", word): + yield match.start(), Name.Variable.Instance, word # lit-word + elif re.match("#.*", word): + yield match.start(), Name.Label, word # issue + elif re.match("%.*", word): + yield match.start(), Name.Decorator, word # file + elif re.match(":.*", word): + yield match.start(), Generic.Subheading, word # get-word + else: + yield match.start(), Name.Variable, word + + tokens = { + 'root': [ + (r'[^R]+', Comment), + (r'Red/System\s+\[', Generic.Strong, 'script'), + (r'Red\s+\[', Generic.Strong, 'script'), + (r'R', Comment) + ], + 'script': [ + (r'\s+', Text), + (r'#"', String.Char, 'char'), + (r'#{[0-9a-f\s]*}', Number.Hex), + (r'2#{', Number.Hex, 'bin2'), + (r'64#{[0-9a-z+/=\s]*}', Number.Hex), + (r'([0-9a-f]+)(h)((\s)|(?=[\[\]{}""\(\)]))', + bygroups(Number.Hex, Name.Variable, Whitespace)), + (r'"', String, 'string'), + (r'{', String, 'string2'), + (r';#+.*\n', Comment.Special), + (r';\*+.*\n', Comment.Preproc), + (r';.*\n', Comment), + (r'%"', Name.Decorator, 'stringFile'), + (r'%[^(\^{^")\s\[\]]+', Name.Decorator), + (r'[+-]?([a-z]{1,3})?\$\d+(\.\d+)?', Number.Float), # money + (r'[+-]?\d+\:\d+(\:\d+)?(\.\d+)?', String.Other), # time + (r'\d+[\-\/][0-9a-z]+[\-\/]\d+(\/\d+\:\d+((\:\d+)?' + r'([\.\d+]?([+-]?\d+:\d+)?)?)?)?', String.Other), # date + (r'\d+(\.\d+)+\.\d+', Keyword.Constant), # tuple + (r'\d+[xX]\d+', Keyword.Constant), # pair + (r'[+-]?\d+(\'\d+)?([\.,]\d*)?[eE][+-]?\d+', Number.Float), + (r'[+-]?\d+(\'\d+)?[\.,]\d*', Number.Float), + (r'[+-]?\d+(\'\d+)?', Number), + (r'[\[\]\(\)]', Generic.Strong), + (r'[a-z]+[^(\^{"\s:)]*://[^(\^{"\s)]*', Name.Decorator), # url + (r'mailto:[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # url + (r'[^(\^{"@\s)]+@[^(\^{"@\s)]+', Name.Decorator), # email + (r'comment\s"', Comment, 'commentString1'), + (r'comment\s{', Comment, 'commentString2'), + (r'comment\s\[', Comment, 'commentBlock'), + (r'comment\s[^(\s{\"\[]+', Comment), + (r'/[^(\^{^")\s/[\]]*', Name.Attribute), + (r'([^(\^{^")\s/[\]]+)(?=[:({"\s/\[\]])', word_callback), + (r'<[\w:.-]*>', Name.Tag), + (r'<[^(<>\s")]+', Name.Tag, 'tag'), + (r'([^(\^{^")\s]+)', Text), + ], + 'string': [ + (r'[^(\^")]+', String), + (escape_re, String.Escape), + (r'[\(|\)]+', String), + (r'\^.', String.Escape), + (r'"', String, '#pop'), + ], + 'string2': [ + (r'[^(\^{^})]+', String), + (escape_re, String.Escape), + (r'[\(|\)]+', String), + (r'\^.', String.Escape), + (r'{', String, '#push'), + (r'}', String, '#pop'), + ], + 'stringFile': [ + (r'[^(\^")]+', Name.Decorator), + (escape_re, Name.Decorator), + (r'\^.', Name.Decorator), + (r'"', Name.Decorator, '#pop'), + ], + 'char': [ + (escape_re + '"', String.Char, '#pop'), + (r'\^."', String.Char, '#pop'), + (r'."', String.Char, '#pop'), + ], + 'tag': [ + (escape_re, Name.Tag), + (r'"', Name.Tag, 'tagString'), + (r'[^(<>\r\n")]+', Name.Tag), + (r'>', Name.Tag, '#pop'), + ], + 'tagString': [ + (r'[^(\^")]+', Name.Tag), + (escape_re, Name.Tag), + (r'[\(|\)]+', Name.Tag), + (r'\^.', Name.Tag), + (r'"', Name.Tag, '#pop'), + ], + 'tuple': [ + (r'(\d+\.)+', Keyword.Constant), + (r'\d+', Keyword.Constant, '#pop'), + ], + 'bin2': [ + (r'\s+', Number.Hex), + (r'([0-1]\s*){8}', Number.Hex), + (r'}', Number.Hex, '#pop'), + ], + 'commentString1': [ + (r'[^(\^")]+', Comment), + (escape_re, Comment), + (r'[\(|\)]+', Comment), + (r'\^.', Comment), + (r'"', Comment, '#pop'), + ], + 'commentString2': [ + (r'[^(\^{^})]+', Comment), + (escape_re, Comment), + (r'[\(|\)]+', Comment), + (r'\^.', Comment), + (r'{', Comment, '#push'), + (r'}', Comment, '#pop'), + ], + 'commentBlock': [ + (r'\[', Comment, '#push'), + (r'\]', Comment, '#pop'), + (r'"', Comment, "commentString1"), + (r'{', Comment, "commentString2"), + (r'[^(\[\]\"{)]+', Comment), + ], + } diff --git a/pygments/lexers/parsers.py b/pygments/lexers/parsers.py index fc8cbb6f..4c23c760 100644 --- a/pygments/lexers/parsers.py +++ b/pygments/lexers/parsers.py @@ -102,7 +102,7 @@ class RagelLexer(RegexLexer): 'host': [ (r'(' + r'|'.join(( # keep host code in largest possible chunks r'[^{}\'"/#]+', # exclude unsafe characters - r'[^\\][\\][{}]', # allow escaped { or } + r'[^\\]\\[{}]', # allow escaped { or } # strings and comments may safely contain unsafe characters r'"(\\\\|\\"|[^"])*"', # double quote string @@ -173,7 +173,7 @@ class RagelEmbeddedLexer(RegexLexer): r'[^}\'"\[/#]', # exclude unsafe characters r'}(?=[^%]|$)', # } is okay as long as it's not followed by % r'}%(?=[^%]|$)', # ...well, one %'s okay, just not two... - r'[^\\][\\][{}]', # ...and } is okay if it's escaped + r'[^\\]\\[{}]', # ...and } is okay if it's escaped # allow / if it's preceded with one of these symbols # (ragel EOF actions) diff --git a/pygments/lexers/qbasic.py b/pygments/lexers/qbasic.py new file mode 100644 index 00000000..80b80f9f --- /dev/null +++ b/pygments/lexers/qbasic.py @@ -0,0 +1,157 @@ +# -*- coding: utf-8 -*- +""" + pygments.lexers.qbasic + ~~~~~~~~~~~~~~~~~~~~~~ + + Simple lexer for Microsoft QBasic source code. + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import re + +from pygments.lexer import RegexLexer, include, bygroups +from pygments.token import Text, Name, Comment, String, Keyword, Punctuation, \ + Number, Operator + +__all__ = ['QBasicLexer'] + + +class QBasicLexer(RegexLexer): + """ + For + `QBasic <http://en.wikipedia.org/wiki/QBasic>`_ + source code. + """ + + name = 'QBasic' + aliases = ['qbasic', 'basic'] + filenames = ['*.BAS', '*.bas'] + mimetypes = ['text/basic'] + + declarations = ['DATA', 'LET'] + + functions = [ + 'ABS', 'ASC', 'ATN', 'CDBL', 'CHR$', 'CINT', 'CLNG', + 'COMMAND$', 'COS', 'CSNG', 'CSRLIN', 'CVD', 'CVDMBF', 'CVI', + 'CVL', 'CVS', 'CVSMBF', 'DATE$', 'ENVIRON$', 'EOF', 'ERDEV', + 'ERDEV$', 'ERL', 'ERR', 'EXP', 'FILEATTR', 'FIX', 'FRE', + 'FREEFILE', 'HEX$', 'INKEY$', 'INP', 'INPUT$', 'INSTR', 'INT', + 'IOCTL$', 'LBOUND', 'LCASE$', 'LEFT$', 'LEN', 'LOC', 'LOF', + 'LOG', 'LPOS', 'LTRIM$', 'MID$', 'MKD$', 'MKDMBF$', 'MKI$', + 'MKL$', 'MKS$', 'MKSMBF$', 'OCT$', 'PEEK', 'PEN', 'PLAY', + 'PMAP', 'POINT', 'POS', 'RIGHT$', 'RND', 'RTRIM$', 'SADD', + 'SCREEN', 'SEEK', 'SETMEM', 'SGN', 'SIN', 'SPACE$', 'SPC', + 'SQR', 'STICK', 'STR$', 'STRIG', 'STRING$', 'TAB', 'TAN', + 'TIME$', 'TIMER', 'UBOUND', 'UCASE$', 'VAL', 'VARPTR', + 'VARPTR$', 'VARSEG' + ] + + metacommands = ['$DYNAMIC', '$INCLUDE', '$STATIC'] + + operators = ['AND', 'EQV', 'IMP', 'NOT', 'OR', 'XOR'] + + statements = [ + 'BEEP', 'BLOAD', 'BSAVE', 'CALL', 'CALL ABSOLUTE', + 'CALL INTERRUPT', 'CALLS', 'CHAIN', 'CHDIR', 'CIRCLE', 'CLEAR', + 'CLOSE', 'CLS', 'COLOR', 'COM', 'COMMON', 'CONST', 'DATA', + 'DATE$', 'DECLARE', 'DEF FN', 'DEF SEG', 'DEFDBL', 'DEFINT', + 'DEFLNG', 'DEFSNG', 'DEFSTR', 'DEF', 'DIM', 'DO', 'LOOP', + 'DRAW', 'END', 'ENVIRON', 'ERASE', 'ERROR', 'EXIT', 'FIELD', + 'FILES', 'FOR', 'NEXT', 'FUNCTION', 'GET', 'GOSUB', 'GOTO', + 'IF', 'THEN', 'INPUT', 'INPUT #', 'IOCTL', 'KEY', 'KEY', + 'KILL', 'LET', 'LINE', 'LINE INPUT', 'LINE INPUT #', 'LOCATE', + 'LOCK', 'UNLOCK', 'LPRINT', 'LSET', 'MID$', 'MKDIR', 'NAME', + 'ON COM', 'ON ERROR', 'ON KEY', 'ON PEN', 'ON PLAY', + 'ON STRIG', 'ON TIMER', 'ON UEVENT', 'ON', 'OPEN', 'OPEN COM', + 'OPTION BASE', 'OUT', 'PAINT', 'PALETTE', 'PCOPY', 'PEN', + 'PLAY', 'POKE', 'PRESET', 'PRINT', 'PRINT #', 'PRINT USING', + 'PSET', 'PUT', 'PUT', 'RANDOMIZE', 'READ', 'REDIM', 'REM', + 'RESET', 'RESTORE', 'RESUME', 'RETURN', 'RMDIR', 'RSET', 'RUN', + 'SCREEN', 'SEEK', 'SELECT CASE', 'SHARED', 'SHELL', 'SLEEP', + 'SOUND', 'STATIC', 'STOP', 'STRIG', 'SUB', 'SWAP', 'SYSTEM', + 'TIME$', 'TIMER', 'TROFF', 'TRON', 'TYPE', 'UEVENT', 'UNLOCK', + 'VIEW', 'WAIT', 'WHILE', 'WEND', 'WIDTH', 'WINDOW', 'WRITE' + ] + + keywords = [ + 'ACCESS', 'ALIAS', 'ANY', 'APPEND', 'AS', 'BASE', 'BINARY', + 'BYVAL', 'CASE', 'CDECL', 'DOUBLE', 'ELSE', 'ELSEIF', 'ENDIF', + 'INTEGER', 'IS', 'LIST', 'LOCAL', 'LONG', 'LOOP', 'MOD', + 'NEXT', 'OFF', 'ON', 'OUTPUT', 'RANDOM', 'SIGNAL', 'SINGLE', + 'STEP', 'STRING', 'THEN', 'TO', 'UNTIL', 'USING', 'WEND' + ] + + tokens = { + 'root': [ + (r'\n+', Text), + (r'\s+', Text.Whitespace), + (r'^(\s*)(\d*)(\s*)(REM .*)$', + bygroups(Text.Whitespace, Name.Label, Text.Whitespace, + Comment.Single)), + (r'^(\s*)(\d+)(\s*)', + bygroups(Text.Whitespace, Name.Label, Text.Whitespace)), + (r'(?=[\s]*)(\w+)(?=[\s]*=)', Name.Variable.Global), + (r'(?=[^"]*)\'.*$', Comment.Single), + (r'"[^\n\"]*"', String.Double), + (r'(END)(\s+)(FUNCTION|IF|SELECT|SUB)', + bygroups(Keyword.Reserved, Text.Whitespace, Keyword.Reserved)), + (r'(DECLARE)(\s+)([A-Z]+)(\s+)(\S+)', + bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable, + Text.Whitespace, Name)), + (r'(DIM)(\s+)(SHARED)(\s+)([^\s\(]+)', + bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable, + Text.Whitespace, Name.Variable.Global)), + (r'(DIM)(\s+)([^\s\(]+)', + bygroups(Keyword.Declaration, Text.Whitespace, Name.Variable.Global)), + (r'^(\s*)([a-zA-Z_]+)(\s*)(\=)', + bygroups(Text.Whitespace, Name.Variable.Global, Text.Whitespace, + Operator)), + (r'(GOTO|GOSUB)(\s+)(\w+\:?)', + bygroups(Keyword.Reserved, Text.Whitespace, Name.Label)), + (r'(SUB)(\s+)(\w+\:?)', + bygroups(Keyword.Reserved, Text.Whitespace, Name.Label)), + include('declarations'), + include('functions'), + include('metacommands'), + include('operators'), + include('statements'), + include('keywords'), + (r'[a-zA-Z_]\w*[\$@#&!]', Name.Variable.Global), + (r'[a-zA-Z_]\w*\:', Name.Label), + (r'\-?\d*\.\d+[@|#]?', Number.Float), + (r'\-?\d+[@|#]', Number.Float), + (r'\-?\d+#?', Number.Integer.Long), + (r'\-?\d+#?', Number.Integer), + (r'!=|==|:=|\.=|<<|>>|[-~+/\\*%=<>&^|?:!.]', Operator), + (r'[\[\]{}(),;]', Punctuation), + (r'[\w]+', Name.Variable.Global), + ], + # can't use regular \b because of X$() + 'declarations': [ + (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, declarations)), + Keyword.Declaration), + ], + 'functions': [ + (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, functions)), + Keyword.Reserved), + ], + 'metacommands': [ + (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, metacommands)), + Keyword.Constant), + ], + 'operators': [ + (r'\b(%s)(?=\(|\b)' % '|'.join(map(re.escape, operators)), Operator.Word), + ], + 'statements': [ + (r'\b(%s)\b' % '|'.join(map(re.escape, statements)), + Keyword.Reserved), + ], + 'keywords': [ + (r'\b(%s)\b' % '|'.join(keywords), Keyword), + ], + } + + def analyse_text(text): + return 0.2 diff --git a/pygments/lexers/shell.py b/pygments/lexers/shell.py index b069b375..df9b56f4 100644 --- a/pygments/lexers/shell.py +++ b/pygments/lexers/shell.py @@ -71,15 +71,15 @@ class BashLexer(RegexLexer): (r'&', Punctuation), (r'\|', Punctuation), (r'\s+', Text), - (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text), (r'\d+(?= |\Z)', Number), + (r'[^=\s\[\]{}()$"\'`\\<&|;]+', Text), (r'\$#?(\w+|.)', Name.Variable), (r'<', Text), ], 'curly': [ (r'}', Keyword, '#pop'), (r':-', Keyword), - (r'[a-zA-Z0-9_]+', Name.Variable), + (r'\w+', Name.Variable), (r'[^}:"\'`$]+', Punctuation), (r':', Punctuation), include('root'), @@ -91,6 +91,8 @@ class BashLexer(RegexLexer): 'math': [ (r'\)\)', Keyword, '#pop'), (r'[-+*/%^|&]|\*\*|\|\|', Operator), + (r'\d+#\d+', Number), + (r'\d+#(?! )', Number), (r'\d+', Number), include('root'), ], @@ -294,24 +296,25 @@ class TcshLexer(RegexLexer): r'umask|unalias|uncomplete|unhash|universe|unlimit|unset|unsetenv|' r'ver|wait|warp|watchlog|where|which)\s*\b', Name.Builtin), - (r'#.*\n', Comment), + (r'#.*', Comment), (r'\\[\w\W]', String.Escape), (r'(\b\w+)(\s*)(=)', bygroups(Name.Variable, Text, Operator)), (r'[\[\]{}()=]+', Operator), (r'<<\s*(\'?)\\?(\w+)[\w\W]+?\2', String), + (r';', Punctuation), ], 'data': [ (r'(?s)"(\\\\|\\[0-7]+|\\.|[^"\\])*"', String.Double), (r"(?s)'(\\\\|\\[0-7]+|\\.|[^'\\])*'", String.Single), (r'\s+', Text), - (r'[^=\s\[\]{}()$"\'`\\]+', Text), + (r'[^=\s\[\]{}()$"\'`\\;#]+', Text), (r'\d+(?= |\Z)', Number), (r'\$#?(\w+|.)', Name.Variable), ], 'curly': [ (r'}', Keyword, '#pop'), (r':-', Keyword), - (r'[a-zA-Z0-9_]+', Name.Variable), + (r'\w+', Name.Variable), (r'[^}:"\'`$]+', Punctuation), (r':', Punctuation), include('root'), @@ -387,13 +390,13 @@ class PowerShellLexer(RegexLexer): (r'`[\'"$@-]', Punctuation), (r'"', String.Double, 'string'), (r"'([^']|'')*'", String.Single), - (r'(\$|@@|@)((global|script|private|env):)?[a-z0-9_]+', + (r'(\$|@@|@)((global|script|private|env):)?\w+', Name.Variable), (r'(%s)\b' % '|'.join(keywords), Keyword), (r'-(%s)\b' % '|'.join(operators), Operator), - (r'(%s)-[a-z_][a-z0-9_]*\b' % '|'.join(verbs), Name.Builtin), - (r'\[[a-z_\[][a-z0-9_. `,\[\]]*\]', Name.Constant), # .net [type]s - (r'-[a-z_][a-z0-9_]*', Name), + (r'(%s)-[a-z_]\w*\b' % '|'.join(verbs), Name.Builtin), + (r'\[[a-z_\[][\w. `,\[\]]*\]', Name.Constant), # .net [type]s + (r'-[a-z_]\w*', Name), (r'\w+', Name), (r'[.,;@{}\[\]$()=+*/\\&%!~?^`|<>-]|::', Punctuation), ], diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 9a3dcb8d..7540f079 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -150,10 +150,10 @@ class PostgresLexer(PostgresBase, RegexLexer): (r"(E|U&)?'(''|[^'])*'", String.Single), (r'(U&)?"(""|[^"])*"', String.Name), # quoted identifier (r'(?s)(\$[^\$]*\$)(.*?)(\1)', language_callback), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), # psql variable in SQL - (r""":(['"]?)[a-z][a-z0-9_]*\b\1""", Name.Variable), + (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable), (r'[;:()\[\]\{\},\.]', Punctuation), ], @@ -192,10 +192,10 @@ class PlPgsqlLexer(PostgresBase, RegexLexer): # Add specific PL/pgSQL rules (before the SQL ones) tokens['root'][:0] = [ - (r'\%[a-z][a-z0-9_]*\b', Name.Builtin), # actually, a datatype + (r'\%[a-z]\w*\b', Name.Builtin), # actually, a datatype (r':=', Operator), - (r'\<\<[a-z][a-z0-9_]*\>\>', Name.Label), - (r'\#[a-z][a-z0-9_]*\b', Keyword.Pseudo), # #variable_conflict + (r'\<\<[a-z]\w*\>\>', Name.Label), + (r'\#[a-z]\w*\b', Keyword.Pseudo), # #variable_conflict ] @@ -219,7 +219,7 @@ class PsqlRegexLexer(PostgresBase, RegexLexer): (r'\n', Text, 'root'), (r'\s+', Text), (r'\\[^\s]+', Keyword.Pseudo), - (r""":(['"]?)[a-z][a-z0-9_]*\b\1""", Name.Variable), + (r""":(['"]?)[a-z]\w*\b\1""", Name.Variable), (r"'(''|[^'])*'", String.Single), (r"`([^`])*`", String.Backtick), (r"[^\s]+", String.Symbol), @@ -435,7 +435,7 @@ class SqlLexer(RegexLexer): # TODO: Backslash escapes? (r"'(''|[^'])*'", String.Single), (r'"(""|[^"])*"', String.Symbol), # not a real string literal in ANSI SQL - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[a-z_]\w*', Name), (r'[;:()\[\],\.]', Punctuation) ], 'multiline-comments': [ @@ -506,10 +506,10 @@ class MySqlLexer(RegexLexer): # TODO: this list is not complete (r'\b(auto_increment|engine|charset|tables)\b', Keyword.Pseudo), (r'(true|false|null)', Name.Constant), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*)(\()', + (r'([a-z_]\w*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name), - (r'@[A-Za-z0-9]*[._]*[A-Za-z0-9]*', Name.Variable), + (r'[a-z_]\w*', Name), + (r'@[a-z0-9]*[._]*[a-z0-9]*', Name.Variable), (r'[;:()\[\],\.]', Punctuation) ], 'multiline-comments': [ @@ -584,7 +584,7 @@ class RqlLexer(RegexLexer): (r'[+*/<>=%-]', Operator), (r'(Any|is|instance_of|CWEType|CWRelation)\b', Name.Builtin), (r'[0-9]+', Number.Integer), - (r'[A-Z_][A-Z0-9_]*\??', Name), + (r'[A-Z_]\w*\??', Name), (r"'(''|[^'])*'", String.Single), (r'"(""|[^"])*"', String.Single), (r'[;:()\[\],\.]', Punctuation) diff --git a/pygments/lexers/templates.py b/pygments/lexers/templates.py index d06ea288..4d53dca6 100644 --- a/pygments/lexers/templates.py +++ b/pygments/lexers/templates.py @@ -161,22 +161,22 @@ class SmartyLexer(RegexLexer): (r'(\{php\})(.*?)(\{/php\})', bygroups(Comment.Preproc, using(PhpLexer, startinline=True), Comment.Preproc)), - (r'(\{)(/?[a-zA-Z_][a-zA-Z0-9_]*)(\s*)', + (r'(\{)(/?[a-zA-Z_]\w*)(\s*)', bygroups(Comment.Preproc, Name.Function, Text), 'smarty'), (r'\{', Comment.Preproc, 'smarty') ], 'smarty': [ (r'\s+', Text), (r'\}', Comment.Preproc, '#pop'), - (r'#[a-zA-Z_][a-zA-Z0-9_]*#', Name.Variable), - (r'\$[a-zA-Z_][a-zA-Z0-9_]*(\.[a-zA-Z0-9_]+)*', Name.Variable), + (r'#[a-zA-Z_]\w*#', Name.Variable), + (r'\$[a-zA-Z_]\w*(\.\w+)*', Name.Variable), (r'[~!%^&*()+=|\[\]:;,.<>/?{}@-]', Operator), (r'(true|false|null)\b', Keyword.Constant), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name.Attribute) + (r'[a-zA-Z_]\w*', Name.Attribute) ] } @@ -207,7 +207,7 @@ class VelocityLexer(RegexLexer): flags = re.MULTILINE | re.DOTALL - identifier = r'[a-zA-Z_][a-zA-Z0-9_]*' + identifier = r'[a-zA-Z_]\w*' tokens = { 'root': [ @@ -267,8 +267,8 @@ class VelocityLexer(RegexLexer): rv += 0.15 if re.search(r'#\{?foreach\}?\(.+?\).*?#\{?end\}?', text): rv += 0.15 - if re.search(r'\$\{?[a-zA-Z_][a-zA-Z0-9_]*(\([^)]*\))?' - r'(\.[a-zA-Z0-9_]+(\([^)]*\))?)*\}?', text): + if re.search(r'\$\{?[a-zA-Z_]\w*(\([^)]*\))?' + r'(\.\w+(\([^)]*\))?)*\}?', text): rv += 0.01 return rv @@ -347,25 +347,25 @@ class DjangoLexer(RegexLexer): Text, Comment.Preproc, Text, Keyword, Text, Comment.Preproc)), # filter blocks - (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\{%)(-?\s*)(filter)(\s+)([a-zA-Z_]\w*)', bygroups(Comment.Preproc, Text, Keyword, Text, Name.Function), 'block'), - (r'(\{%)(-?\s*)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\{%)(-?\s*)([a-zA-Z_]\w*)', bygroups(Comment.Preproc, Text, Keyword), 'block'), (r'\{', Other) ], 'varnames': [ - (r'(\|)(\s*)([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(\|)(\s*)([a-zA-Z_]\w*)', bygroups(Operator, Text, Name.Function)), - (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_][a-zA-Z0-9_]*)', + (r'(is)(\s+)(not)?(\s+)?([a-zA-Z_]\w*)', bygroups(Keyword, Text, Keyword, Text, Name.Function)), (r'(_|true|false|none|True|False|None)\b', Keyword.Pseudo), (r'(in|as|reversed|recursive|not|and|or|is|if|else|import|' r'with(?:(?:out)?\s*context)?|scoped|ignore\s+missing)\b', Keyword), (r'(loop|block|super|forloop)\b', Name.Builtin), - (r'[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable), - (r'\.[a-zA-Z0-9_]+', Name.Variable), + (r'[a-zA-Z][\w-]*', Name.Variable), + (r'\.\w+', Name.Variable), (r':?"(\\\\|\\"|[^"])*"', String.Double), (r":?'(\\\\|\\'|[^'])*'", String.Single), (r'([{}()\[\]+\-*/,:~]|[><=]=?)', Operator), @@ -745,7 +745,7 @@ class CheetahLexer(RegexLexer): (bygroups(Comment.Preproc, using(CheetahPythonLexer), Comment.Preproc))), # TODO support other Python syntax like $foo['bar'] - (r'(\$)([a-zA-Z_][a-zA-Z0-9_\.]*[a-zA-Z0-9_])', + (r'(\$)([a-zA-Z_][\w\.]*\w)', bygroups(Comment.Preproc, using(CheetahPythonLexer))), (r'(\$\{!?)(.*?)(\})(?s)', bygroups(Comment.Preproc, using(CheetahPythonLexer), @@ -753,7 +753,7 @@ class CheetahLexer(RegexLexer): (r'''(?sx) (.+?) # anything, followed by: (?: - (?=[#][#a-zA-Z]*) | # an eval comment + (?=\#[#a-zA-Z]*) | # an eval comment (?=\$[a-zA-Z_{]) | # a substitution \Z # end of string ) @@ -843,7 +843,7 @@ class GenshiTextLexer(RegexLexer): 'variable': [ (r'(?<!\$)(\$\{)(.+?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), - (r'(?<!\$)(\$)([a-zA-Z_][a-zA-Z0-9_\.]*)', + (r'(?<!\$)(\$)([a-zA-Z_][\w\.]*)', Name.Variable), ] } @@ -871,7 +871,7 @@ class GenshiMarkupLexer(RegexLexer): ], 'pytag': [ (r'\s+', Text), - (r'[a-zA-Z0-9_:-]+\s*=', Name.Attribute, 'pyattr'), + (r'[\w:-]+\s*=', Name.Attribute, 'pyattr'), (r'/?\s*>', Name.Tag, '#pop'), ], 'pyattr': [ @@ -881,8 +881,8 @@ class GenshiMarkupLexer(RegexLexer): ], 'tag': [ (r'\s+', Text), - (r'py:[a-zA-Z0-9_-]+\s*=', Name.Attribute, 'pyattr'), - (r'[a-zA-Z0-9_:-]+\s*=', Name.Attribute, 'attr'), + (r'py:[\w-]+\s*=', Name.Attribute, 'pyattr'), + (r'[\w:-]+\s*=', Name.Attribute, 'attr'), (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ @@ -907,7 +907,7 @@ class GenshiMarkupLexer(RegexLexer): 'variable': [ (r'(?<!\$)(\$\{)(.+?)(\})', bygroups(Comment.Preproc, using(PythonLexer), Comment.Preproc)), - (r'(?<!\$)(\$)([a-zA-Z_][a-zA-Z0-9_\.]*)', + (r'(?<!\$)(\$)([a-zA-Z_][\w\.]*)', Name.Variable), ] } @@ -1498,13 +1498,15 @@ class ColdfusionLexer(RegexLexer): # strings, evidently. (r"'.*?'", String.Single), (r'\d+', Number), - (r'(if|else|len|var|case|default|break|switch|component|property|function|do|try|catch|in|continue|for|return|while)\b', Keyword), - (r'(required|any|array|binary|boolean|component|date|guid|numeric|query|string|struct|uuid|xml)\b', Keyword), + (r'(if|else|len|var|xml|default|break|switch|component|property|function|do|' + r'try|catch|in|continue|for|return|while|required|any|array|binary|boolean|' + r'component|date|guid|numeric|query|string|struct|uuid|case)\b', Keyword), (r'(true|false|null)\b', Keyword.Constant), - (r'(application|session|client|cookie|super|this|variables|arguments)\b', Name.Constant), - (r'([A-Za-z_$][A-Za-z0-9_.]*)(\s*)(\()', + (r'(application|session|client|cookie|super|this|variables|arguments)\b', + Name.Constant), + (r'([a-z_$][\w.]*)(\s*)(\()', bygroups(Name.Function, Text, Punctuation)), - (r'[A-Za-z_$][A-Za-z0-9_.]*', Name.Variable), + (r'[a-z_$][\w.]*', Name.Variable), (r'[()\[\]{};:,.\\]', Punctuation), (r'\s+', Text), ], @@ -1534,7 +1536,7 @@ class ColdfusionMarkupLexer(RegexLexer): (r'<[^<>]*', Other), ], 'tags': [ - (r'(?s)<!---.*?--->', Comment.Multiline), + (r'<!---', Comment.Multiline, 'cfcomment'), (r'(?s)<!--.*?-->', Comment), (r'<cfoutput.*?>', Name.Builtin, 'cfoutput'), (r'(?s)(<cfscript.*?>)(.+?)(</cfscript.*?>)', @@ -1556,6 +1558,12 @@ class ColdfusionMarkupLexer(RegexLexer): (r'(?s)<[^<>]*', Other), (r'#', Other), ], + 'cfcomment': [ + (r'(?s)(.*?)(<!---)', + bygroups(Comment.Multiline, Comment.Multiline), '#push'), + (r'(?s)(.*?)(--->)', + bygroups(Comment.Multiline, Comment.Multiline), '#pop'), + ], } @@ -1803,8 +1811,8 @@ class HandlebarsLexer(RegexLexer): # borrowed from DjangoLexer (r':?"(\\\\|\\"|[^"])*"', String.Double), (r":?'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z][a-zA-Z0-9_-]*', Name.Variable), - (r'\.[a-zA-Z0-9_]+', Name.Variable), + (r'[a-zA-Z][\w-]*', Name.Variable), + (r'\.\w+', Name.Variable), (r"[0-9](\.[0-9]*)?(eE[+-][0-9])?[flFLdD]?|" r"0[xX][0-9a-fA-F]+[Ll]?", Number), ] diff --git a/pygments/lexers/text.py b/pygments/lexers/text.py index db3200b3..10675654 100644 --- a/pygments/lexers/text.py +++ b/pygments/lexers/text.py @@ -17,6 +17,7 @@ from pygments.lexer import Lexer, LexerContext, RegexLexer, ExtendedRegexLexer, from pygments.token import Punctuation, Text, Comment, Keyword, Name, String, \ Generic, Operator, Number, Whitespace, Literal from pygments.util import get_bool_opt, ClassNotFound +from pygments.lexers.agile import PythonLexer from pygments.lexers.other import BashLexer __all__ = ['IniLexer', 'PropertiesLexer', 'SourcesListLexer', 'BaseMakefileLexer', @@ -26,7 +27,7 @@ __all__ = ['IniLexer', 'PropertiesLexer', 'SourcesListLexer', 'BaseMakefileLexer 'DebianControlLexer', 'DarcsPatchLexer', 'YamlLexer', 'LighttpdConfLexer', 'NginxConfLexer', 'CMakeLexer', 'HttpLexer', 'PyPyLogLexer', 'RegeditLexer', 'HxmlLexer', 'EbnfLexer', - 'TodotxtLexer'] + 'TodotxtLexer', 'DockerLexer'] class IniLexer(RegexLexer): @@ -234,11 +235,11 @@ class BaseMakefileLexer(RegexLexer): (r'\$[<@$+%?|*]', Keyword), (r'\s+', Text), (r'#.*?\n', Comment), - (r'(export)(\s+)(?=[a-zA-Z0-9_${}\t -]+\n)', + (r'(export)(\s+)(?=[\w${}\t -]+\n)', bygroups(Keyword, Text), 'export'), (r'export\s+', Keyword), # assignment - (r'([a-zA-Z0-9_${}.-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)', + (r'([\w${}.-]+)(\s*)([!?:+]?=)([ \t]*)((?:.*\\\n)+|.*\n)', bygroups(Name.Variable, Text, Operator, Text, using(BashLexer))), # strings (r'(?s)"(\\\\|\\.|[^"\\])*"', String.Double), @@ -257,7 +258,7 @@ class BaseMakefileLexer(RegexLexer): (r'\)', Keyword, '#pop'), ], 'export': [ - (r'[a-zA-Z0-9_${}-]+', Name.Variable), + (r'[\w${}-]+', Name.Variable), (r'\n', Text, '#pop'), (r'\s+', Text), ], @@ -588,7 +589,7 @@ class ApacheConfLexer(RegexLexer): (r'(#.*?)$', Comment), (r'(<[^\s>]+)(?:(\s+)(.*?))?(>)', bygroups(Name.Tag, Text, String, Name.Tag)), - (r'([a-zA-Z][a-zA-Z0-9_]*)(\s+)', + (r'([a-z]\w*)(\s+)', bygroups(Name.Builtin, Text), 'value'), (r'\.+', Text), ], @@ -597,7 +598,7 @@ class ApacheConfLexer(RegexLexer): (r'[^\S\n]+', Text), (r'\d+\.\d+\.\d+\.\d+(?:/\d+)?', Number), (r'\d+', Number), - (r'/([a-zA-Z0-9][a-zA-Z0-9_./-]+)', String.Other), + (r'/([a-z0-9][\w./-]+)', String.Other), (r'(on|off|none|any|all|double|email|dns|min|minimal|' r'os|productonly|full|emerg|alert|crit|error|warn|' r'notice|info|debug|registry|script|inetd|standalone|' @@ -775,7 +776,7 @@ class RstLexer(RegexLexer): (r'^( *)(:.*?:)([ \t]+)(.*?)$', bygroups(Text, Name.Class, Text, Name.Function)), # Definition list - (r'^([^\s].*(?<!::)\n)((?:(?: +.*)\n)+)', + (r'^(\S.*(?<!::)\n)((?:(?: +.*)\n)+)', bygroups(using(this, state='inline'), using(this, state='inline'))), # Code blocks (r'(::)(\n[ \t]*\n)([ \t]+)(.*)(\n)((?:(?:\3.*|)\n)+)', @@ -835,8 +836,16 @@ class VimLexer(RegexLexer): mimetypes = ['text/x-vim'] flags = re.MULTILINE + _python = r'py(?:t(?:h(?:o(?:n)?)?)?)?' + tokens = { 'root': [ + (r'^([ \t:]*)(' + _python + r')([ \t]*)(<<)([ \t]*)(.*)((?:\n|.)*)(\6)', + bygroups(using(this), Keyword, Text, Operator, Text, Text, + using(PythonLexer), Text)), + (r'^([ \t:]*)(' + _python + r')([ \t])(.*)', + bygroups(using(this), Keyword, Text, using(PythonLexer))), + (r'^\s*".*', Comment), (r'[ \t]+', Text), @@ -1681,6 +1690,12 @@ class CMakeLexer(RegexLexer): ] } + def analyse_text(text): + exp = r'^ *CMAKE_MINIMUM_REQUIRED *\( *VERSION *\d(\.\d)* *( FATAL_ERROR)? *\) *$' + if re.search(exp, text, flags=re.MULTILINE | re.IGNORECASE): + return 0.8 + return 0.0 + class HttpLexer(RegexLexer): """ @@ -2011,3 +2026,30 @@ class TodotxtLexer(RegexLexer): ('\s+', IncompleteTaskText), ], } + + +class DockerLexer(RegexLexer): + """ + Lexer for `Docker <http://docker.io>`_ configuration files. + + .. versionadded:: 2.0 + """ + name = 'Docker' + aliases = ['docker', 'dockerfile'] + filenames = ['Dockerfile', '*.docker'] + mimetypes = ['text/x-dockerfile-config'] + + _keywords = (r'(?:FROM|MAINTAINER|RUN|CMD|EXPOSE|ENV|ADD|ENTRYPOINT|' + r'VOLUME|WORKDIR)') + + flags = re.IGNORECASE | re.MULTILINE + + tokens = { + 'root': [ + (r'^(ONBUILD)(\s+)(%s)\b' % (_keywords,), + bygroups(Name.Keyword, Whitespace, Keyword)), + (_keywords + r'\b', Keyword), + (r'#.*', Comment), + (r'.+', using(BashLexer)), + ], + } diff --git a/pygments/lexers/web.py b/pygments/lexers/web.py index f8c89204..2428ffcd 100644 --- a/pygments/lexers/web.py +++ b/pygments/lexers/web.py @@ -81,7 +81,7 @@ class JavascriptLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -184,8 +184,8 @@ class ActionScriptLexer(RegexLexer): name = 'ActionScript' aliases = ['as', 'actionscript'] filenames = ['*.as'] - mimetypes = ['application/x-actionscript3', 'text/x-actionscript3', - 'text/actionscript3'] + mimetypes = ['application/x-actionscript', 'text/x-actionscript', + 'text/actionscript'] flags = re.DOTALL tokens = { @@ -248,7 +248,7 @@ class ActionScriptLexer(RegexLexer): r'isXMLName|clearInterval|fscommand|getTimer|getURL|getVersion|' r'isFinite|parseFloat|parseInt|setInterval|trace|updateAfterEvent|' r'unescape)\b',Name.Function), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-f]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -268,10 +268,10 @@ class ActionScript3Lexer(RegexLexer): name = 'ActionScript 3' aliases = ['as3', 'actionscript3'] filenames = ['*.as'] - mimetypes = ['application/x-actionscript', 'text/x-actionscript', - 'text/actionscript'] + mimetypes = ['application/x-actionscript3', 'text/x-actionscript3', + 'text/actionscript3'] - identifier = r'[$a-zA-Z_][a-zA-Z0-9_]*' + identifier = r'[$a-zA-Z_]\w*' typeidentifier = identifier + '(?:\.<\w+>)?' flags = re.DOTALL | re.MULTILINE @@ -359,11 +359,11 @@ class CssLexer(RegexLexer): (r'\s+', Text), (r'/\*(?:.|\n)*?\*/', Comment), (r'{', Punctuation, 'content'), - (r'\:[a-zA-Z0-9_-]+', Name.Decorator), - (r'\.[a-zA-Z0-9_-]+', Name.Class), - (r'\#[a-zA-Z0-9_-]+', Name.Function), - (r'@[a-zA-Z0-9_-]+', Keyword, 'atrule'), - (r'[a-zA-Z0-9_-]+', Name.Tag), + (r'\:[\w-]+', Name.Decorator), + (r'\.[\w-]+', Name.Class), + (r'\#[\w-]+', Name.Function), + (r'@[\w-]+', Keyword, 'atrule'), + (r'[\w-]+', Name.Tag), (r'[~\^\*!%&$\[\]\(\)<>\|+=@:;,./?-]', Operator), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single) @@ -475,7 +475,7 @@ class CssLexer(RegexLexer): (r'[\[\]();]+', Punctuation), (r'"(\\\\|\\"|[^"])*"', String.Double), (r"'(\\\\|\\'|[^'])*'", String.Single), - (r'[a-zA-Z_][a-zA-Z0-9_]*', Name) + (r'[a-zA-Z_]\w*', Name) ] } @@ -595,26 +595,26 @@ class ObjectiveJLexer(RegexLexer): r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), - (r'([$a-zA-Z_][a-zA-Z0-9_]*)(' + _ws + r')(?=\()', + (r'([$a-zA-Z_]\w*)(' + _ws + r')(?=\()', bygroups(Name.Function, using(this))), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name), + (r'[$a-zA-Z_]\w*', Name), ], 'classname' : [ # interface definition that inherits - (r'([a-zA-Z_][a-zA-Z0-9_]*)(' + _ws + r':' + _ws + - r')([a-zA-Z_][a-zA-Z0-9_]*)?', + (r'([a-zA-Z_]\w*)(' + _ws + r':' + _ws + + r')([a-zA-Z_]\w*)?', bygroups(Name.Class, using(this), Name.Class), '#pop'), # interface definition for a category - (r'([a-zA-Z_][a-zA-Z0-9_]*)(' + _ws + r'\()([a-zA-Z_][a-zA-Z0-9_]*)(\))', + (r'([a-zA-Z_]\w*)(' + _ws + r'\()([a-zA-Z_]\w*)(\))', bygroups(Name.Class, using(this), Name.Label, Text), '#pop'), # simple interface / implementation - (r'([a-zA-Z_][a-zA-Z0-9_]*)', Name.Class, '#pop'), + (r'([a-zA-Z_]\w*)', Name.Class, '#pop'), ], 'forward_classname' : [ - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*,\s*)', + (r'([a-zA-Z_]\w*)(\s*,\s*)', bygroups(Name.Class, Text), '#push'), - (r'([a-zA-Z_][a-zA-Z0-9_]*)(\s*;?)', + (r'([a-zA-Z_]\w*)(\s*;?)', bygroups(Name.Class, Text), '#pop'), ], 'function_signature': [ @@ -622,26 +622,26 @@ class ObjectiveJLexer(RegexLexer): # start of a selector w/ parameters (r'(\(' + _ws + r')' # open paren - r'([a-zA-Z_][a-zA-Z0-9_]+)' # return type + r'([a-zA-Z_]\w+)' # return type r'(' + _ws + r'\)' + _ws + r')' # close paren - r'([$a-zA-Z_][a-zA-Z0-9_]+' + _ws + r':)', # function name + r'([$a-zA-Z_]\w+' + _ws + r':)', # function name bygroups(using(this), Keyword.Type, using(this), Name.Function), 'function_parameters'), # no-param function (r'(\(' + _ws + r')' # open paren - r'([a-zA-Z_][a-zA-Z0-9_]+)' # return type + r'([a-zA-Z_]\w+)' # return type r'(' + _ws + r'\)' + _ws + r')' # close paren - r'([$a-zA-Z_][a-zA-Z0-9_]+)', # function name + r'([$a-zA-Z_]\w+)', # function name bygroups(using(this), Keyword.Type, using(this), Name.Function), "#pop"), # no return type given, start of a selector w/ parameters - (r'([$a-zA-Z_][a-zA-Z0-9_]+' + _ws + r':)', # function name + (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name bygroups (Name.Function), 'function_parameters'), # no return type given, no-param function - (r'([$a-zA-Z_][a-zA-Z0-9_]+)', # function name + (r'([$a-zA-Z_]\w+)', # function name bygroups(Name.Function), "#pop"), ('', Text, '#pop'), @@ -653,11 +653,11 @@ class ObjectiveJLexer(RegexLexer): (r'(\(' + _ws + ')' # open paren r'([^\)]+)' # type r'(' + _ws + r'\)' + _ws + r')' # close paren - r'([$a-zA-Z_][a-zA-Z0-9_]+)', # param name + r'([$a-zA-Z_]\w+)', # param name bygroups(using(this), Keyword.Type, using(this), Text)), # one piece of a selector name - (r'([$a-zA-Z_][a-zA-Z0-9_]+' + _ws + r':)', # function name + (r'([$a-zA-Z_]\w+' + _ws + r':)', # function name Name.Function), # smallest possible selector piece @@ -667,10 +667,10 @@ class ObjectiveJLexer(RegexLexer): (r'(,' + _ws + r'\.\.\.)', using(this)), # param name - (r'([$a-zA-Z_][a-zA-Z0-9_]+)', Text), + (r'([$a-zA-Z_]\w+)', Text), ], 'expression' : [ - (r'([$a-zA-Z_][a-zA-Z0-9_]*)(\()', bygroups(Name.Function, + (r'([$a-zA-Z_]\w*)(\()', bygroups(Name.Function, Punctuation)), (r'(\))', Punctuation, "#pop"), ], @@ -737,8 +737,8 @@ class HtmlLexer(RegexLexer): ], 'tag': [ (r'\s+', Text), - (r'[a-zA-Z0-9_:-]+\s*=', Name.Attribute, 'attr'), - (r'[a-zA-Z0-9_:-]+', Name.Attribute), + (r'[\w:-]+\s*=', Name.Attribute, 'attr'), + (r'[\w:-]+', Name.Attribute), (r'/?\s*>', Name.Tag, '#pop'), ], 'script-content': [ @@ -801,8 +801,8 @@ class PhpLexer(RegexLexer): # Note that a backslash is included in the following two patterns # PHP uses a backslash as a namespace separator - _ident_char = r'[\\_a-zA-Z0-9]|[^\x00-\x7f]' - _ident_begin = r'(?:[\\_a-zA-Z]|[^\x00-\x7f])' + _ident_char = r'[\\\w]|[^\x00-\x7f]' + _ident_begin = r'(?:[\\_a-z]|[^\x00-\x7f])' _ident_end = r'(?:' + _ident_char + ')*' _ident_inner = _ident_begin + _ident_end @@ -852,7 +852,7 @@ class PhpLexer(RegexLexer): (r'(\d+\.\d*|\d*\.\d+)([eE][+-]?[0-9]+)?', Number.Float), (r'\d+[eE][+-]?[0-9]+', Number.Float), (r'0[0-7]+', Number.Oct), - (r'0[xX][a-fA-F0-9]+', Number.Hex), + (r'0[xX][a-f0-9]+', Number.Hex), (r'\d+', Number.Integer), (r'0b[01]+', Number.Binary), (r"'([^'\\]*(?:\\.[^'\\]*)*)'", String.Single), @@ -868,7 +868,7 @@ class PhpLexer(RegexLexer): 'string': [ (r'"', String.Double, '#pop'), (r'[^{$"\\]+', String.Double), - (r'\\([nrt\"$\\]|[0-7]{1,3}|x[0-9A-Fa-f]{1,2})', String.Escape), + (r'\\([nrt\"$\\]|[0-7]{1,3}|x[0-9a-f]{1,2})', String.Escape), (r'\$' + _ident_inner + '(\[\S+?\]|->' + _ident_inner + ')?', String.Interpol), (r'(\{\$\{)(.*?)(\}\})', @@ -1118,8 +1118,8 @@ class MxmlLexer(RegexLexer): ('<!--', Comment, 'comment'), (r'<\?.*?\?>', Comment.Preproc), ('<![^>]*>', Comment.Preproc), - (r'<\s*[a-zA-Z0-9:._-]+', Name.Tag, 'tag'), - (r'<\s*/\s*[a-zA-Z0-9:._-]+\s*>', Name.Tag), + (r'<\s*[\w:.-]+', Name.Tag, 'tag'), + (r'<\s*/\s*[\w:.-]+\s*>', Name.Tag), ], 'comment': [ ('[^-]+', Comment), @@ -1128,7 +1128,7 @@ class MxmlLexer(RegexLexer): ], 'tag': [ (r'\s+', Text), - (r'[a-zA-Z0-9_.:-]+\s*=', Name.Attribute, 'attr'), + (r'[\w.:-]+\s*=', Name.Attribute, 'attr'), (r'/?\s*>', Name.Tag, '#pop'), ], 'attr': [ @@ -1161,11 +1161,10 @@ class HaxeLexer(ExtendedRegexLexer): r'inline|using|null|true|false|abstract)\b') # idtype in lexer.mll - typeid = r'_*[A-Z][_a-zA-Z0-9]*' + typeid = r'_*[A-Z]\w*' # combined ident and dollar and idtype - ident = r'(?:_*[a-z][_a-zA-Z0-9]*|_+[0-9][_a-zA-Z0-9]*|' + typeid + \ - '|_+|\$[_a-zA-Z0-9]+)' + ident = r'(?:_*[a-z]\w*|_+[0-9]\w*|' + typeid + '|_+|\$\w+)' binop = (r'(?:%=|&=|\|=|\^=|\+=|\-=|\*=|/=|<<=|>\s*>\s*=|>\s*>\s*>\s*=|==|' r'!=|<=|>\s*=|&&|\|\||<<|>>>|>\s*>|\.\.\.|<|>|%|&|\||\^|\+|\*|' @@ -1297,6 +1296,19 @@ class HaxeLexer(ExtendedRegexLexer): (r'\(', Comment.Preproc, ('#pop', 'preproc-parenthesis')), (ident, Comment.Preproc, '#pop'), + + # Float + (r'\.[0-9]+', Number.Float), + (r'[0-9]+[eE][\+\-]?[0-9]+', Number.Float), + (r'[0-9]+\.[0-9]*[eE][\+\-]?[0-9]+', Number.Float), + (r'[0-9]+\.[0-9]+', Number.Float), + (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float), + + # Int + (r'0x[0-9a-fA-F]+', Number.Hex), + (r'[0-9]+', Number.Integer), + + # String (r"'", String.Single, ('#pop', 'string-single')), (r'"', String.Double, ('#pop', 'string-double')), ], @@ -1321,6 +1333,19 @@ class HaxeLexer(ExtendedRegexLexer): ('#pop', 'preproc-expr-chain', 'preproc-parenthesis')), (ident, Comment.Preproc, ('#pop', 'preproc-expr-chain')), + + # Float + (r'\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+[eE][\+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.[0-9]*[eE][\+\-]?[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.[0-9]+', Number.Float, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+\.(?!' + ident + '|\.\.)', Number.Float, ('#pop', 'preproc-expr-chain')), + + # Int + (r'0x[0-9a-fA-F]+', Number.Hex, ('#pop', 'preproc-expr-chain')), + (r'[0-9]+', Number.Integer, ('#pop', 'preproc-expr-chain')), + + # String (r"'", String.Single, ('#pop', 'preproc-expr-chain', 'string-single')), (r'"', String.Double, @@ -1454,7 +1479,7 @@ class HaxeLexer(ExtendedRegexLexer): 'class-member': [ include('spaces'), (r'(var)\b', Keyword.Declaration, - ('#pop', 'optional-semicolon', 'prop')), + ('#pop', 'optional-semicolon', 'var')), (r'(function)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'class-method')), ], @@ -1463,7 +1488,7 @@ class HaxeLexer(ExtendedRegexLexer): 'function-local': [ include('spaces'), (r'(' + ident_no_keyword + ')?', Name.Function, - ('#pop', 'expr', 'flag', 'function-param', + ('#pop', 'optional-expr', 'flag', 'function-param', 'parenthesis-open', 'type-param-constraint')), ], @@ -1495,13 +1520,6 @@ class HaxeLexer(ExtendedRegexLexer): (r',', Punctuation, ('#pop', 'function-param')), ], - # class property - # eg. var prop(default, null):String; - 'prop': [ - include('spaces'), - (ident_no_keyword, Name, ('#pop', 'assign', 'flag', 'prop-get-set')), - ], - 'prop-get-set': [ include('spaces'), (r'\(', Punctuation, ('#pop', 'parenthesis-close', @@ -1528,7 +1546,8 @@ class HaxeLexer(ExtendedRegexLexer): 'meta-ident', 'meta-colon')), (r'(?:\+\+|\-\-|~(?!/)|!|\-)', Operator), (r'\(', Punctuation, ('#pop', 'expr-chain', 'parenthesis')), - (r'(?:inline)\b', Keyword.Declaration), + (r'(?:static|public|private|override|dynamic|inline)\b', + Keyword.Declaration), (r'(?:function)\b', Keyword.Declaration, ('#pop', 'expr-chain', 'function-local')), (r'\{', Punctuation, ('#pop', 'expr-chain', 'bracket')), @@ -1587,7 +1606,15 @@ class HaxeLexer(ExtendedRegexLexer): # macro reification 'macro': [ include('spaces'), + include('meta'), (r':', Punctuation, ('#pop', 'type')), + + (r'(?:extern|private)\b', Keyword.Declaration), + (r'(?:abstract)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'abstract')), + (r'(?:class|interface)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'class')), + (r'(?:enum)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'enum')), + (r'(?:typedef)\b', Keyword.Declaration, ('#pop', 'optional-semicolon', 'typedef')), + (r'', Text, ('#pop', 'expr')), ], @@ -1740,7 +1767,7 @@ class HaxeLexer(ExtendedRegexLexer): 'dollar': [ include('spaces'), - (r'\{', Keyword, ('#pop', 'bracket-close', 'expr')), + (r'\{', Punctuation, ('#pop', 'expr-chain', 'bracket-close', 'expr')), (r'', Text, ('#pop', 'expr-chain')), ], @@ -1868,7 +1895,7 @@ class HaxeLexer(ExtendedRegexLexer): # a parenthesis expr that contain exactly one expr 'parenthesis': [ include('spaces'), - (r'', Text, ('#pop', 'parenthesis-close', 'expr')), + (r'', Text, ('#pop', 'parenthesis-close', 'flag', 'expr')), ], 'parenthesis-open': [ @@ -1883,7 +1910,7 @@ class HaxeLexer(ExtendedRegexLexer): 'var': [ include('spaces'), - (ident_no_keyword, Text, ('#pop', 'var-sep', 'assign', 'flag')), + (ident_no_keyword, Text, ('#pop', 'var-sep', 'assign', 'flag', 'prop-get-set')), ], # optional more var decl. @@ -2038,8 +2065,8 @@ class HamlLexer(ExtendedRegexLexer): ], 'css': [ - (r'\.[a-z0-9_:-]+', Name.Class, 'tag'), - (r'\#[a-z0-9_:-]+', Name.Function, 'tag'), + (r'\.[\w:-]+', Name.Class, 'tag'), + (r'\#[\w:-]+', Name.Function, 'tag'), ], 'eval-or-plain': [ @@ -2052,7 +2079,7 @@ class HamlLexer(ExtendedRegexLexer): 'content': [ include('css'), - (r'%[a-z0-9_:-]+', Name.Tag, 'tag'), + (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), @@ -2088,16 +2115,16 @@ class HamlLexer(ExtendedRegexLexer): 'html-attributes': [ (r'\s+', Text), - (r'[a-z0-9_:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), - (r'[a-z0-9_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), + (r'[\w:-]+', Name.Attribute), (r'\)', Text, '#pop'), ], 'html-attribute-value': [ (r'[ \t]+', Text), - (r'[a-z0-9_]+', Name.Variable, '#pop'), - (r'@[a-z0-9_]+', Name.Variable.Instance, '#pop'), - (r'\$[a-z0-9_]+', Name.Variable.Global, '#pop'), + (r'\w+', Name.Variable, '#pop'), + (r'@\w+', Name.Variable.Instance, '#pop'), + (r'\$\w+', Name.Variable.Global, '#pop'), (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'), (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'), ], @@ -2237,7 +2264,7 @@ common_sass_tokens = { (r'\:', Name.Decorator, 'pseudo-class'), (r'\.', Name.Class, 'class'), (r'\#', Name.Namespace, 'id'), - (r'[a-zA-Z0-9_-]+', Name.Tag), + (r'[\w-]+', Name.Tag), (r'#\{', String.Interpol, 'interpolation'), (r'&', Keyword), (r'[~\^\*!&\[\]\(\)<>\|+=@:;,./?-]', Operator), @@ -2317,7 +2344,7 @@ class SassLexer(ExtendedRegexLexer): (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'), (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'), (r'@extend', Keyword, 'selector'), - (r'@[a-z0-9_-]+', Keyword, 'selector'), + (r'@[\w-]+', Keyword, 'selector'), (r'=[\w-]+', Name.Function, 'value'), (r'\+[\w-]+', Name.Decorator, 'value'), (r'([!$][\w-]\w*)([ \t]*(?:(?:\|\|)?=|:))', @@ -2390,7 +2417,7 @@ class ScssLexer(RegexLexer): (r'(@mixin)( [\w-]+)', bygroups(Keyword, Name.Function), 'value'), (r'(@include)( [\w-]+)', bygroups(Keyword, Name.Decorator), 'value'), (r'@extend', Keyword, 'selector'), - (r'@[a-z0-9_-]+', Keyword, 'selector'), + (r'@[\w-]+', Keyword, 'selector'), (r'(\$[\w-]*\w)([ \t]*:)', bygroups(Name.Variable, Operator), 'value'), (r'(?=[^;{}][;}])', Name.Attribute, 'attr'), (r'(?=[^;{}:]+:[^a-z])', Name.Attribute, 'attr'), @@ -2473,12 +2500,12 @@ class CoffeeScriptLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'eval|isFinite|isNaN|parseFloat|parseInt|document|window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\.:\$]*\s*[:=]\s', Name.Variable, + (r'[$a-zA-Z_][\w\.:\$]*\s*[:=]\s', Name.Variable, 'slashstartsregex'), - (r'@[$a-zA-Z_][a-zA-Z0-9_\.:\$]*\s*[:=]\s', Name.Variable.Instance, + (r'@[$a-zA-Z_][\w\.:\$]*\s*[:=]\s', Name.Variable.Instance, 'slashstartsregex'), (r'@', Name.Other, 'slashstartsregex'), - (r'@?[$a-zA-Z_][a-zA-Z0-9_\$]*', Name.Other, 'slashstartsregex'), + (r'@?[$a-zA-Z_][\w\$]*', Name.Other, 'slashstartsregex'), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -2500,6 +2527,7 @@ class CoffeeScriptLexer(RegexLexer): (r'"', String, '#pop'), (r'\\.|\'', String), # double-quoted string don't need ' escapes (r'#{', String.Interpol, "interpoling_string"), + (r'#', String), include('strings') ], 'sqs': [ @@ -2511,6 +2539,7 @@ class CoffeeScriptLexer(RegexLexer): (r'"""', String, '#pop'), (r'\\.|\'|"', String), # no need to escape quotes in triple-string (r'#{', String.Interpol, "interpoling_string"), + (r'#', String), include('strings'), ], 'tsqs': [ @@ -2544,13 +2573,13 @@ class KalLexer(RegexLexer): (r'#(?!##[^#]).*?\n', Comment.Single), ], 'functiondef': [ - (r'[$a-zA-Z_][a-zA-Z0-9_\$]*\s*', Name.Function, '#pop'), + (r'[$a-zA-Z_][\w\$]*\s*', Name.Function, '#pop'), include('commentsandwhitespace'), ], 'classdef': [ (r'\binherits\s+from\b', Keyword), - (r'[$a-zA-Z_][a-zA-Z0-9_\$]*\s*\n', Name.Class, '#pop'), - (r'[$a-zA-Z_][a-zA-Z0-9_\$]*\s*', Name.Class), + (r'[$a-zA-Z_][\w\$]*\s*\n', Name.Class, '#pop'), + (r'[$a-zA-Z_][\w\$]*\s*', Name.Class), include('commentsandwhitespace'), ], 'listcomprehension': [ @@ -2579,7 +2608,7 @@ class KalLexer(RegexLexer): (r'\b(function|method|task)\b', Keyword.Declaration, 'functiondef'), (r'\bclass\b', Keyword.Declaration, 'classdef'), (r'\b(safe\s+)?wait\s+for\b', Keyword, 'waitfor'), - (r'\b(me|this)(\.[$a-zA-Z_][a-zA-Z0-9_\.\$]*)?\b', Name.Variable.Instance), + (r'\b(me|this)(\.[$a-zA-Z_][\w\.\$]*)?\b', Name.Variable.Instance), (r'(?<![\.\$])(for(\s+(parallel|series))?|in|of|while|until|' r'break|return|continue|' r'when|if|unless|else|otherwise|except\s+when|' @@ -2595,7 +2624,7 @@ class KalLexer(RegexLexer): r'eval|isFinite|isNaN|parseFloat|parseInt|document|window|' r'print)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\.\$]*\s*(:|[\+\-\*\/]?\=)?\b', Name.Variable), + (r'[$a-zA-Z_][\w\.\$]*\s*(:|[\+\-\*\/]?\=)?\b', Name.Variable), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -2698,19 +2727,19 @@ class LiveScriptLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'eval|isFinite|isNaN|parseFloat|parseInt|document|window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\.\-:\$]*\s*[:=]\s', Name.Variable, + (r'[$a-zA-Z_][\w\.\-:\$]*\s*[:=]\s', Name.Variable, 'slashstartsregex'), - (r'@[$a-zA-Z_][a-zA-Z0-9_\.\-:\$]*\s*[:=]\s', Name.Variable.Instance, + (r'@[$a-zA-Z_][\w\.\-:\$]*\s*[:=]\s', Name.Variable.Instance, 'slashstartsregex'), (r'@', Name.Other, 'slashstartsregex'), - (r'@?[$a-zA-Z_][a-zA-Z0-9_\-]*', Name.Other, 'slashstartsregex'), + (r'@?[$a-zA-Z_][\w\-]*', Name.Other, 'slashstartsregex'), (r'[0-9]+\.[0-9]+([eE][0-9]+)?[fd]?(?:[a-zA-Z_]+)?', Number.Float), (r'[0-9]+(~[0-9a-z]+)?(?:[a-zA-Z_]+)?', Number.Integer), ('"""', String, 'tdqs'), ("'''", String, 'tsqs'), ('"', String, 'dqs'), ("'", String, 'sqs'), - (r'\\[\w$-]+', String), + (r'\\\S+', String), (r'<\[.*?\]>', String), ], 'strings': [ @@ -2808,8 +2837,8 @@ class ScamlLexer(ExtendedRegexLexer): ], 'css': [ - (r'\.[a-z0-9_:-]+', Name.Class, 'tag'), - (r'\#[a-z0-9_:-]+', Name.Function, 'tag'), + (r'\.[\w:-]+', Name.Class, 'tag'), + (r'\#[\w:-]+', Name.Function, 'tag'), ], 'eval-or-plain': [ @@ -2822,7 +2851,7 @@ class ScamlLexer(ExtendedRegexLexer): 'content': [ include('css'), - (r'%[a-z0-9_:-]+', Name.Tag, 'tag'), + (r'%[\w:-]+', Name.Tag, 'tag'), (r'!!!' + _dot + r'*\n', Name.Namespace, '#pop'), (r'(/)(\[' + _dot + '*?\])(' + _dot + r'*\n)', bygroups(Comment, Comment.Special, Comment), @@ -2861,16 +2890,16 @@ class ScamlLexer(ExtendedRegexLexer): 'html-attributes': [ (r'\s+', Text), - (r'[a-z0-9_:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), - (r'[a-z0-9_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), + (r'[\w:-]+', Name.Attribute), (r'\)', Text, '#pop'), ], 'html-attribute-value': [ (r'[ \t]+', Text), - (r'[a-z0-9_]+', Name.Variable, '#pop'), - (r'@[a-z0-9_]+', Name.Variable.Instance, '#pop'), - (r'\$[a-z0-9_]+', Name.Variable.Global, '#pop'), + (r'\w+', Name.Variable, '#pop'), + (r'@\w+', Name.Variable.Instance, '#pop'), + (r'\$\w+', Name.Variable.Global, '#pop'), (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'), (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'), ], @@ -2918,8 +2947,8 @@ class JadeLexer(ExtendedRegexLexer): ], 'css': [ - (r'\.[a-z0-9_:-]+', Name.Class, 'tag'), - (r'\#[a-z0-9_:-]+', Name.Function, 'tag'), + (r'\.[\w:-]+', Name.Class, 'tag'), + (r'\#[\w:-]+', Name.Function, 'tag'), ], 'eval-or-plain': [ @@ -2947,7 +2976,7 @@ class JadeLexer(ExtendedRegexLexer): '#pop'), (r':' + _dot + r'*\n', _starts_block(Name.Decorator, 'filter-block'), '#pop'), - (r'[a-z0-9_:-]+', Name.Tag, 'tag'), + (r'[\w:-]+', Name.Tag, 'tag'), (r'\|', Text, 'eval-or-plain'), ], @@ -2970,16 +2999,16 @@ class JadeLexer(ExtendedRegexLexer): 'html-attributes': [ (r'\s+', Text), - (r'[a-z0-9_:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), - (r'[a-z0-9_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'html-attribute-value'), + (r'[\w:-]+', Name.Attribute), (r'\)', Text, '#pop'), ], 'html-attribute-value': [ (r'[ \t]+', Text), - (r'[a-z0-9_]+', Name.Variable, '#pop'), - (r'@[a-z0-9_]+', Name.Variable.Instance, '#pop'), - (r'\$[a-z0-9_]+', Name.Variable.Global, '#pop'), + (r'\w+', Name.Variable, '#pop'), + (r'@\w+', Name.Variable.Instance, '#pop'), + (r'\$\w+', Name.Variable.Global, '#pop'), (r"'(\\\\|\\'|[^'\n])*'", String, '#pop'), (r'"(\\\\|\\"|[^"\n])*"', String, '#pop'), ], @@ -3699,8 +3728,8 @@ class DartLexer(RegexLexer): (r'\b(bool|double|Dynamic|int|num|Object|String|void)\b', Keyword.Type), (r'\b(false|null|true)\b', Keyword.Constant), (r'[~!%^&*+=|?:<>/-]|as\b', Operator), - (r'[a-zA-Z_$][a-zA-Z0-9_]*:', Name.Label), - (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_$]\w*:', Name.Label), + (r'[a-zA-Z_$]\w*', Name), (r'[(){}\[\],.;]', Punctuation), (r'0[xX][0-9a-fA-F]+', Number.Hex), # DIGIT+ (‘.’ DIGIT*)? EXPONENT? @@ -3710,13 +3739,13 @@ class DartLexer(RegexLexer): # pseudo-keyword negate intentionally left out ], 'class': [ - (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name.Class, '#pop') + (r'[a-zA-Z_$]\w*', Name.Class, '#pop') ], 'import_decl': [ include('string_literal'), (r'\s+', Text), (r'\b(as|show|hide)\b', Keyword), - (r'[a-zA-Z_$][a-zA-Z0-9_]*', Name), + (r'[a-zA-Z_$]\w*', Name), (r'\,', Punctuation), (r'\;', Punctuation, '#pop') ], @@ -3735,7 +3764,7 @@ class DartLexer(RegexLexer): 'string_common': [ (r"\\(x[0-9A-Fa-f]{2}|u[0-9A-Fa-f]{4}|u\{[0-9A-Fa-f]*\}|[a-z\'\"$\\])", String.Escape), - (r'(\$)([a-zA-Z_][a-zA-Z0-9_]*)', bygroups(String.Interpol, Name)), + (r'(\$)([a-zA-Z_]\w*)', bygroups(String.Interpol, Name)), (r'(\$\{)(.*?)(\})', bygroups(String.Interpol, using(this), String.Interpol)) ], @@ -3818,7 +3847,7 @@ class TypeScriptLexer(RegexLexer): r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), # Match stuff like: module name {...} - (r'\b(module)(\s*)(\s*[a-zA-Z0-9_?.$][\w?.$]*)(\s*)', + (r'\b(module)(\s*)(\s*[\w?.$][\w?.$]*)(\s*)', bygroups(Keyword.Reserved, Text, Name.Other, Text), 'slashstartsregex'), # Match variable type keywords (r'\b(string|bool|number)\b', Keyword.Type), @@ -3830,9 +3859,9 @@ class TypeScriptLexer(RegexLexer): # Match stuff like: function() {...} (r'([a-zA-Z_?.$][\w?.$]*)\(\) \{', Name.Other, 'slashstartsregex'), # Match stuff like: (function: return type) - (r'([a-zA-Z0-9_?.$][\w?.$]*)(\s*:\s*)([a-zA-Z0-9_?.$][\w?.$]*)', + (r'([\w?.$][\w?.$]*)(\s*:\s*)([\w?.$][\w?.$]*)', bygroups(Name.Other, Text, Keyword.Type)), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -4165,7 +4194,7 @@ class QmlLexer(RegexLexer): r'decodeURIComponent|encodeURI|encodeURIComponent|' r'Error|eval|isFinite|isNaN|parseFloat|parseInt|document|this|' r'window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_]*', Name.Other), + (r'[$a-zA-Z_]\w*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), @@ -4256,8 +4285,8 @@ class MaskLexer(RegexLexer): (r'"', String, 'string-double'), (r'([\w-]+)', Name.Tag, 'node'), (r'([^\.#;{>\s]+)', Name.Class, 'node'), - (r'(#[\w_-]+)', Name.Function, 'node'), - (r'(\.[\w_-]+)', Name.Variable.Class, 'node') + (r'(#[\w-]+)', Name.Function, 'node'), + (r'(\.[\w-]+)', Name.Variable.Class, 'node') ], 'string-base': [ (r'\\.', String.Escape), @@ -4294,8 +4323,8 @@ class MaskLexer(RegexLexer): (r'\.', Name.Variable.Class, 'node-class'), (r'\#', Name.Function, 'node-id'), (r'style[ \t]*=', Name.Attribute, 'node-attr-style-value'), - (r'[\w_:-]+[ \t]*=', Name.Attribute, 'node-attr-value'), - (r'[\w_:-]+', Name.Attribute), + (r'[\w:-]+[ \t]*=', Name.Attribute, 'node-attr-value'), + (r'[\w:-]+', Name.Attribute), (r'[>{;]', Punctuation, '#pop') ], 'node-class': [ @@ -4310,7 +4339,7 @@ class MaskLexer(RegexLexer): ], 'node-attr-value':[ (r'\s+', Text), - (r'[\w_]+', Name.Variable, '#pop'), + (r'\w+', Name.Variable, '#pop'), (r"'", String, 'string-single-pop2'), (r'"', String, 'string-double-pop2'), (r'', Text, '#pop') @@ -4323,8 +4352,8 @@ class MaskLexer(RegexLexer): ], 'css-base': [ (r'\s+', Text), - (r"[;]", Punctuation), - (r"[\w\-_]+\s*:", Name.Builtin) + (r";", Punctuation), + (r"[\w\-]+\s*:", Name.Builtin) ], 'css-single-end': [ include('css-base'), @@ -4388,19 +4417,20 @@ class ZephirLexer(RegexLexer): r'(<<|>>>?|==?|!=?|->|[-<>+*%&\|\^/])=?', Operator, 'slashstartsregex'), (r'[{(\[;,]', Punctuation, 'slashstartsregex'), (r'[})\].]', Punctuation), - (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|require|inline|' - r'throw|try|catch|finally|new|delete|typeof|instanceof|void|namespace|use|extends|' - r'this|fetch|isset|unset|echo|fetch|likely|unlikely|empty)\b', Keyword, 'slashstartsregex'), + (r'(for|in|while|do|break|return|continue|switch|case|default|if|else|loop|' + r'require|inline|throw|try|catch|finally|new|delete|typeof|instanceof|void|' + r'namespace|use|extends|this|fetch|isset|unset|echo|fetch|likely|unlikely|' + r'empty)\b', Keyword, 'slashstartsregex'), (r'(var|let|with|function)\b', Keyword.Declaration, 'slashstartsregex'), - (r'(abstract|boolean|bool|char|class|const|double|enum|export|' - r'extends|final|float|goto|implements|import|int|string|interface|long|ulong|char|uchar|native|unsigned|' - r'private|protected|public|short|static|self|throws|reverse|' + (r'(abstract|boolean|bool|char|class|const|double|enum|export|extends|final|' + r'native|goto|implements|import|int|string|interface|long|ulong|char|uchar|' + r'float|unsigned|private|protected|public|short|static|self|throws|reverse|' r'transient|volatile)\b', Keyword.Reserved), (r'(true|false|null|undefined)\b', Keyword.Constant), (r'(Array|Boolean|Date|_REQUEST|_COOKIE|_SESSION|' r'_GET|_POST|_SERVER|this|stdClass|range|count|iterator|' r'window)\b', Name.Builtin), - (r'[$a-zA-Z_][a-zA-Z0-9_\\]*', Name.Other), + (r'[$a-zA-Z_][\w\\]*', Name.Other), (r'[0-9][0-9]*\.[0-9]+([eE][0-9]+)?[fd]?', Number.Float), (r'0x[0-9a-fA-F]+', Number.Hex), (r'[0-9]+', Number.Integer), diff --git a/pygments/styles/xcode.py b/pygments/styles/xcode.py index e2ecf2aa..8bb2c24e 100644 --- a/pygments/styles/xcode.py +++ b/pygments/styles/xcode.py @@ -11,7 +11,7 @@ from pygments.style import Style from pygments.token import Keyword, Name, Comment, String, Error, \ - Number, Operator + Number, Operator, Literal class XcodeStyle(Style): @@ -30,13 +30,13 @@ class XcodeStyle(Style): Operator: '#000000', - Keyword: '#AA0D92', + Keyword: '#A90D91', Name: '#000000', Name.Attribute: '#836C28', - Name.Class: '#000000', + Name.Class: '#3F6E75', Name.Function: '#000000', - Name.Builtin: '#AA0D92', + Name.Builtin: '#A90D91', # In Obj-C code this token is used to colour Cocoa types Name.Builtin.Pseudo: '#5B269A', Name.Variable: '#000000', @@ -45,6 +45,7 @@ class XcodeStyle(Style): # Workaround for a BUG here: lexer treats multiline method signatres as labels Name.Label: '#000000', - Number: '#2300CE', + Literal: '#1C01CE', + Number: '#1C01CE', Error: '#000000', } diff --git a/pygments/util.py b/pygments/util.py index c302900f..5dc6981f 100644 --- a/pygments/util.py +++ b/pygments/util.py @@ -208,6 +208,11 @@ def looks_like_xml(text): # Python narrow build compatibility def _surrogatepair(c): + # Given a unicode character code + # with length greater than 16 bits, + # return the two 16 bit surrogate pair. + # From example D28 of: + # http://www.unicode.org/book/ch03.pdf return (0xd7c0 + (c >> 10), (0xdc00 + (c & 0x3ff))) def unirange(a, b): diff --git a/tests/examplefiles/demo.cfm b/tests/examplefiles/demo.cfm index d94a06a0..49690484 100644 --- a/tests/examplefiles/demo.cfm +++ b/tests/examplefiles/demo.cfm @@ -1,4 +1,11 @@ <!--- cfcomment ---> +<!--- nested <!--- cfcomment ---> ---> +<!--- multi-line +nested +<!--- +cfcomment +---> +---> <!-- html comment --> <html> <head> @@ -17,6 +24,9 @@ #IsDate("foo")#<br /> #DaysInMonth(RightNow)# </cfoutput> +<cfset x="x"> +<cfset y="y"> +<cfset z="z"> <cfoutput group="x"> #x# <cfoutput>#y#</cfoutput> @@ -29,6 +39,8 @@ <cfset greeting = "Hello #person#"> <cfset greeting = "Hello" & " world!"> +<cfset a = 5> +<cfset b = 10> <cfset c = a^b> <cfset c = a MOD b> <cfset c = a / b> diff --git a/tests/examplefiles/example.chai b/tests/examplefiles/example.chai new file mode 100644 index 00000000..85f53c38 --- /dev/null +++ b/tests/examplefiles/example.chai @@ -0,0 +1,6 @@ +var f = fun(x) { x + 2; } +// comment +puts(someFunc(2 + 2 - 1 * 5 / 4)); +var x = "str"; +def dosomething(lhs, rhs) { print("lhs: ${lhs}, rhs: ${rhs}"); } +callfunc(`+`, 1, 4); diff --git a/tests/examplefiles/function_arrows.coffee b/tests/examplefiles/example.coffee index cd1ef1e8..2cbd1df3 100644 --- a/tests/examplefiles/function_arrows.coffee +++ b/tests/examplefiles/example.coffee @@ -1,3 +1,5 @@ +# function arrows + methodA:-> 'A' methodB:=> 'B' methodC:()=> 'C' @@ -9,3 +11,17 @@ methodF:(c,d)-> 'F' (-> 'I') (=> 'J') + +# strings + +"#{wow}" +"w#{wow}w" +"#wow" +"wow#" +"w#ow" + +'#{wow}' +'w#{wow}w' +'#wow' +'wow#' +'w#ow' diff --git a/tests/examplefiles/example.f90 b/tests/examplefiles/example.f90 new file mode 100644 index 00000000..40462189 --- /dev/null +++ b/tests/examplefiles/example.f90 @@ -0,0 +1,8 @@ +program main + integer, parameter :: mykind = selected_real_kind() + print *, 1 + print *, 1_mykind + print *, 1. + print *, 1._mykind + print *, (1., 1._mykind) +end program main diff --git a/tests/examplefiles/example.feature b/tests/examplefiles/example.feature new file mode 100644 index 00000000..a26268da --- /dev/null +++ b/tests/examplefiles/example.feature @@ -0,0 +1,16 @@ +# First comment +Feature: My amazing feature + Feature description line 1 + Feature description line 2 + +#comment +Scenario Outline: My detailed scenario #string + Given That <x> is set + When When I <subtract> + Then I should get the <remain#der> + + # indented comment + Examples: + | x | subtract | remain#der | + | 12 | 5\|3 | #73 | + | #the | 10 | 15 | diff --git a/tests/examplefiles/example.groovy b/tests/examplefiles/example.groovy new file mode 100755 index 00000000..25ef2eab --- /dev/null +++ b/tests/examplefiles/example.groovy @@ -0,0 +1,2 @@ +#!/usr/bin/env groovy +println "Hello World" diff --git a/tests/examplefiles/example.hs b/tests/examplefiles/example.hs new file mode 100644 index 00000000..9efd3364 --- /dev/null +++ b/tests/examplefiles/example.hs @@ -0,0 +1,27 @@ +module ĈrazyThings where + +import "base" Data.Char +import "base" Data.Char (isControl, isSpace) +import "base" Data.Char (isControl, --isSpace) + isSpace) +import "base" Data.Char (isControl, -- isSpace) + isSpace) + +(-->) :: Num a => a -- signature +(-->) = 2 -- >implementation + +--test comment +-- test comment + +main :: IO () +main = putStrLn "hello world" + +gádd x y = x + y +ádd x y = x + y + + +data ĈrazyThings = + Ĉar | + House | + Peár + deriving (Show, Eq) diff --git a/tests/examplefiles/example.hx b/tests/examplefiles/example.hx index fd93bb49..381cf825 100644 --- a/tests/examplefiles/example.hx +++ b/tests/examplefiles/example.hx @@ -139,4 +139,47 @@ typedef Pt2 = { y:Float, ?z:Float, //optional z add : Point -> Void, -}
\ No newline at end of file +} + + +//top-level class members +public function test(); +private var attr(get, set) = 1; + + +//pre-proc number +public static inline function indexOf<T>(arr:Array<T>, v:T) : Int +{ + #if (haxe_ver >= 3.1) + return arr.indexOf(v); + #else + #if (flash || js) + return untyped arr.indexOf(v); + #else + return std.Lambda.indexOf(arr, v); + #end + #end +} + +//macro reification +var e = macro var $myVar = 0; +var e = macro ${v}.toLowerCase(); +var e = macro o.$myField; +var e = macro { $myField : 0 }; +var e = macro $i{varName}++; +var e = macro $v{myStr}; +var args = [macro "sub", macro 3]; +var e = macro "Hello".toLowerCase($a{args}); +(macro $i{tmp}.addAtom($v{name}, $atom)).finalize(op.pos); + +var c = macro class MyClass { + public function new() { } + public function $funcName() { + trace($v{funcName} + " was called"); + } +} + +var c = macro interface IClass {}; + +//ECheckType +var f = (123:Float);
\ No newline at end of file diff --git a/tests/examplefiles/example.j b/tests/examplefiles/example.j new file mode 100644 index 00000000..16cdde86 --- /dev/null +++ b/tests/examplefiles/example.j @@ -0,0 +1,564 @@ +; Example JVM assembly +; Tested with JasminXT 2.4 + +.bytecode 49.0 +.source HelloWorld.java +.class public final enum HelloWorld +.super java/lang/Object +.implements java/io/Serializable +.signature "Ljava/lang/Object;Ljava/io/Serializable;" +.enclosing method hw/jasmin.HelloWorldRunner.run()V +.deprecated +.annotation visible HelloWorld + I I = 0 +.end annotation +.debug "Happy debugging!" + +.inner interface public InnerInterface inner 'HelloWorld$InnerInterface' outer HelloWorld +.inner class public InnerClass inner HelloWorld$InnerClass outer 'HelloWorld' + +.field public volatile transient I I +.field static protected final serialVersionUID 'J' signature "TJ;" = 2147483648 +.field annotation protected 'protected' [[[Lcom/oracle/util/Checksums; + .deprecated + .signature "[[[Lcom/oracle/util/Checksums;" + .attribute foo "foo.txt" + .attribute 'foo' "foo.txt" +.end field +.field public newline I +.field public static defaultString 'Ljava/lang/String;' + +.method public <init>()V + .limit stack 3 +.line 7 + .var 0 is self LHelloWorld; from 0 to 1 + aload_0 + invokenonvirtual java/lang/Object/<init>()V + return +.end method + +.method static public main([Ljava/lang/String;)V + .limit locals 7 + .limit stack 10 + .throws java.lang/RuntimeException + .catch java/lang.ClassCastException from cast to 'extra_l' using /extra + .signature "([Ljava/lang/String;)V" + .stack + offset /Input + locals Object java/lang/String + locals Uninitialized 'End' + locals Uninitialized 0 + locals Top + locals Integer + locals Float + locals Long + locals Double + locals Null + locals UninitializedThis + stack Object java/lang/String + stack Uninitialized End + stack 'Uninitialized' 0 + stack 'Top' + stack Integer + stack Float + stack Long + stack Double + stack Null + stack UninitializedThis + .end stack + .stack use 1 locals + offset 'extra' + .end stack + .stack use locals + .end stack +.line 0xd + .var 0 is args [Ljava/lang/String; + aload_w 0 + arraylength + ifne /Input + iconst_1 + anewarray java/lang/String + checkcast [Ljava/lang/String; + astore_0 + aload_0 + iconst_0 + ldc "World" + dup + putstatic HelloWorld.defaultString Ljava/lang/String; + aastore +/Input: + iconst_2 + iconst_3 + multianewarray [[C 2 + astore_1 + aload_1 + iconst_0 + aaload + astore_2 + aload_1 + iconst_1 + aaload + astore_3 + +<<o: + aload_3 + iconst_0 + invokestatic HelloWorld/int()I + castore + +<<\u0020: + aload_3 + dconst_1 + dconst_0 + dsub + d2i + invokestatic HelloWorld/double()D + d2i + castore + +<<!: + aload_3 + lconst_0 + dup2 + lxor + lconst_1 + dup2 + ladd + lsub + lneg + l2i + invokestatic HelloWorld/long()J + l2i + castore + +<<H: + aload_2 + fconst_0 + fconst_1 + fconst_2 + dup_x2 + fdiv + fmul + f2l + l2i + swap + invokestatic HelloWorld/float(F)F + f2i + castore + +<<e : + aload_2 + iconst_1 + i2s + i2c + i2b + iconst_1 + newarray short + dup + iconst_0 + iconst_1 + newarray byte + dup + iconst_0 + sipush 0x65 + bastore + iconst_0 + baload + sastore + iconst_0 + saload + int2short + int2char + int2byte + castore + + <<l : + aload_2 + iconst_2 + bipush 0x1b +*2: + iconst_1 + ishl + dup + lookupswitch + 0: '/lookupswitch' + 0x6c: /lookupswitch + default: *2 +/lookupswitch: + castore + + ldc2_w 2 + dup2 + lcmp + .set i 4 + .set 'j' 5 + .var 4 is i I from 'i++' to End + .var 5 is j I signature "I" from i++ to End + istore 4 + goto 1 +i++: + iinc 4 1 +1: iconst_0 + istore_w 5 + goto_w 2 +j++: + iinc_w 5 1 +2: getstatic java/lang/System/out Ljava/io/PrintStream; + aload_1 + iload 4 + aaload + iload_w 5 + caload + invokevirtual java/io/PrintStream/print(C)V + iload 5 + iconst_1 + if_icmpne $+6 + jsr extra + iload 5 + iconst_2 + if_icmplt j++ + iconst_1 + iload 4 + if_icmpgt i++ + +<<\u00a0: + getstatic java/lang/System/out Ljava/io/PrintStream; + invokestatic HelloWorld/get"example()LHelloWorld; + getfield HelloWorld/newline I + invokevirtual java/io/PrintStream/print(C)V +End: + return + +extra: + astore 6 + iload 4 + tableswitch 0 1 + extra_l + extra_string + default: 'End' + nop +extra_string: + getstatic java/lang/System/out Ljava/io/PrintStream; + aload 0 + iconst_0 + aaload + invokevirtual java/io/PrintStream/print(Ljava/lang/String;)V +cast: + ldc java/lang/String + checkcast java/lang/Class + pop + ldc Ljava/lang/String; + checkcast Ljava/lang/Class; + pop + iconst_1 + dup + newarray boolean + checkcast [Z + pop + newarray 'int' + checkcast HelloWorld + checkcast LHelloWorld; + pop +extra_l: + getstatic java/lang/System/out Ljava/io/PrintStream; + dup + ldc "\123.\456.\u006c.\n.\r.\t.\f.\b.\".\'.\\" + iconst_5 + invokeinterface java/lang/CharSequence/charAt(I)C 2 + invokevirtual java/io/PrintStream/print(C)V +/extra: + pop + ret 6 +.end method + +.method private static get"example()LHelloWorld; + .limit locals 3 + .limit stack 4 + .catch all from 7 to 53 using 59 + aconst_null + dup + dup + astore_w 0 +try: + goto $+0x11 +finally: + astore_w 2 + putfield HelloWorld/newline I + ret_w 2 + nop + aload_0 + areturn + ifnonnull $-2 + ifnull $+3 + new HelloWorld + dup + dup + invokespecial HelloWorld/<init>()V + astore 0 + aload 0 + monitorenter + monitorexit + new java/lang/RuntimeException + dup + invokespecial java/lang/RuntimeException/<init>()V + athrow + aconst_null +/try: + dup + aconst_null + if_acmpeq $+3 + areturn +catch: + jsr $+10 + aload_0 + dup + aconst_null + if_acmpne /try + areturn + astore_1 + aload_0 + ldc 10 + jsr_w finally + ret 1 +'single\u0020quoted\u0020label': ; Messes up [@ below if lexed sloppily +.end method + +.method varargs private static int()I + .annotation invisible HelloWorld + [@ [@ WhatIsThis??? = .annotation ; name, type, exttype + I I = 1 ; name, type + another-I I = 2 + Enum e Ljava/util/logging/Level; = FINE + .end annotation + .annotation + s s = "foo" + another-s s = "bar" + Enum [e Ljava/util/logging/Level; = FINE FINE 'FINE' FINE + .end annotation + float F = 123.456 + .end annotation + .annotation visibleparam 1 LHelloWorld; + x [I = 0x01 0x02 0x03 + y I = 2 + .end annotation + .annotation invisibleparam 255 HelloWorld + a F = 1.2 + b D = 3.4 + .end annotation + .annotation default + I = 0 + .end annotation + .limit locals 4 + .limit stack 20 + iconst_1 + newarray int + dup + dup + instanceof [Z + bipush 0x9 + bipush 0xB + iand + iconst_5 + iconst_4 + dup_x1 + iconst_m1 + iadd + bipush +-111 + ineg + swap + idiv + dup_x2 + dup + ishr + ishl + imul + ior + bipush -73 + ixor + isub + dup + iconst_1 + iadd + irem + iastore + iconst_0 + iaload + istore_0 + iload_0 + istore_1 + iload_1 + istore_2 + iload_2 + istore_3 + iload_3 + dup + dup + dup2_x1 + if_icmpeq $+33 + dup + dup + if_icmpge $+28 + dup + dup + if_icmple $+23 + dup + ifle $+19 + dup + ifeq $+15 + dup + iflt $+11 + dup + ifgt $+7 + dup + ifge $+3 + ireturn +.end method + +.method static private fpstrict double()D + .limit locals 7 + .limit stack 11 + dconst_1 + dconst_0 + dcmpg + newarray double + dup + dconst_0 + dup2 + dcmpl + ldc2_w 128. + ldc2_w -240.221d + dneg + ldc2_w 158.d + dup2 + dadd + dup2_x2 + drem + ddiv + pop2 + dconst_1 + dmul + d2f + f2d + d2l + l2i + iconst_2 + iushr + i2d + dastore + iconst_0 + daload + dstore_0 + dload_0 + dstore_1 + dload_1 + dstore_2 + dload_2 + dstore_3 + dload_3 + dstore 4 + dload 4 + dstore_w 5 + dload_w 5 + dreturn +.end method + +.method static long()J + .limit locals 7 + .limit stack 11 + iconst_1 + newarray long + dup + iconst_0 + ldc2_w 5718613688 + ldc2_w 3143486100 + ldc2_w 0x3 + ldiv + lmul + ldc2_w -10000000000 + lrem + ldc_w 0x60 + i2l + lor + ldc 0x33 + i2l + land + dup2 + iconst_1 + lshl + iconst_3 + lshr + iconst_3 + lushr + ladd + l2d + d2l + l2f + f2l + lastore + iconst_0 + laload + lstore_0 + lload_0 + lstore_1 + lload_1 + lstore_2 + lload_2 + lstore_3 + lload_3 + lstore 4 + lload 4 + lstore_w 5 + lload_w 5 + lreturn +.end method + +.method private static float(F)F + .limit locals 6 + .limit stack 9 + iconst_1 + newarray float + dup + fload_0 + dup + fcmpg + fload_0 + dup + dup + dup + dup2_x2 + fadd + fsub + fneg + frem + ldc 70 + i2f + fadd + fadd + swap + pop + fastore + fload_0 + dup + fcmpl + faload + fstore_0 + fload_0 + fstore_1 + fload_1 + fstore_2 + fload_2 + fstore_3 + fload_3 + fstore 4 + fload 4 + fstore_w 5 + fload_w 5 + freturn +.end method + +.method abstract bridge synthetic 'acc1()V' + breakpoint +.end method + +.method native synchronized acc2()V +.end method diff --git a/tests/examplefiles/example.java b/tests/examplefiles/example.java new file mode 100644 index 00000000..f2e94322 --- /dev/null +++ b/tests/examplefiles/example.java @@ -0,0 +1,16 @@ +class _PostUnico$deClassá +{void fo$o() {} + + void PostUnicodeFunctioná() { + láb$el: + break láb$el; + + } +} + +class áPreUnicode$Class +{ + public int $foo; + public int á$foo; + _PostUnico$deClassá áPreUnicodeFunction() { return null; } +} diff --git a/tests/examplefiles/example.pp b/tests/examplefiles/example.pp new file mode 100644 index 00000000..ea697be2 --- /dev/null +++ b/tests/examplefiles/example.pp @@ -0,0 +1,8 @@ +exec { 'grep': + command => 'grep "\'" -rI *', + path => '/bin:/usr/bin', +} + +node default { + notify {"Hello World":;} +} diff --git a/tests/examplefiles/example.red b/tests/examplefiles/example.red new file mode 100644 index 00000000..37c17ef8 --- /dev/null +++ b/tests/examplefiles/example.red @@ -0,0 +1,257 @@ +Red [
+ Title: "Red console"
+ Author: ["Nenad Rakocevic" "Kaj de Vos"]
+ File: %console.red
+ Tabs: 4
+ Rights: "Copyright (C) 2012-2013 Nenad Rakocevic. All rights reserved."
+ License: {
+ Distributed under the Boost Software License, Version 1.0.
+ See https://github.com/dockimbel/Red/blob/master/BSL-License.txt
+ }
+ Purpose: "Just some code for testing Pygments colorizer"
+ Language: http://www.red-lang.org/
+]
+
+#system-global [
+ #either OS = 'Windows [
+ #import [
+ "kernel32.dll" stdcall [
+ AttachConsole: "AttachConsole" [
+ processID [integer!]
+ return: [integer!]
+ ]
+ SetConsoleTitle: "SetConsoleTitleA" [
+ title [c-string!]
+ return: [integer!]
+ ]
+ ReadConsole: "ReadConsoleA" [
+ consoleInput [integer!]
+ buffer [byte-ptr!]
+ charsToRead [integer!]
+ numberOfChars [int-ptr!]
+ inputControl [int-ptr!]
+ return: [integer!]
+ ]
+ ]
+ ]
+ line-buffer-size: 16 * 1024
+ line-buffer: allocate line-buffer-size
+ ][
+ #switch OS [
+ MacOSX [
+ #define ReadLine-library "libreadline.dylib"
+ ]
+ #default [
+ #define ReadLine-library "libreadline.so.6"
+ #define History-library "libhistory.so.6"
+ ]
+ ]
+ #import [
+ ReadLine-library cdecl [
+ read-line: "readline" [ ; Read a line from the console.
+ prompt [c-string!]
+ return: [c-string!]
+ ]
+ rl-bind-key: "rl_bind_key" [
+ key [integer!]
+ command [integer!]
+ return: [integer!]
+ ]
+ rl-insert: "rl_insert" [
+ count [integer!]
+ key [integer!]
+ return: [integer!]
+ ]
+ ]
+ #if OS <> 'MacOSX [
+ History-library cdecl [
+ add-history: "add_history" [ ; Add line to the history.
+ line [c-string!]
+ ]
+ ]
+ ]
+ ]
+
+ rl-insert-wrapper: func [
+ [cdecl]
+ count [integer!]
+ key [integer!]
+ return: [integer!]
+ ][
+ rl-insert count key
+ ]
+
+ ]
+]
+
+Windows?: system/platform = 'Windows
+
+read-argument: routine [
+ /local
+ args [str-array!]
+ str [red-string!]
+][
+ if system/args-count <> 2 [
+ SET_RETURN(none-value)
+ exit
+ ]
+ args: system/args-list + 1 ;-- skip binary filename
+ str: simple-io/read-txt args/item
+ SET_RETURN(str)
+]
+
+init-console: routine [
+ str [string!]
+ /local
+ ret
+][
+ #either OS = 'Windows [
+ ;ret: AttachConsole -1
+ ;if zero? ret [print-line "ReadConsole failed!" halt]
+
+ ret: SetConsoleTitle as c-string! string/rs-head str
+ if zero? ret [print-line "SetConsoleTitle failed!" halt]
+ ][
+ rl-bind-key as-integer tab as-integer :rl-insert-wrapper
+ ]
+]
+
+input: routine [
+ prompt [string!]
+ /local
+ len ret str buffer line
+][
+ #either OS = 'Windows [
+ len: 0
+ print as c-string! string/rs-head prompt
+ ret: ReadConsole stdin line-buffer line-buffer-size :len null
+ if zero? ret [print-line "ReadConsole failed!" halt]
+ len: len + 1
+ line-buffer/len: null-byte
+ str: string/load as c-string! line-buffer len
+ ][
+ line: read-line as c-string! string/rs-head prompt
+ if line = null [halt] ; EOF
+
+ #if OS <> 'MacOSX [add-history line]
+
+ str: string/load line 1 + length? line
+; free as byte-ptr! line
+ ]
+ SET_RETURN(str)
+]
+
+count-delimiters: function [
+ buffer [string!]
+ return: [block!]
+][
+ list: copy [0 0]
+ c: none
+
+ foreach c buffer [
+ case [
+ escaped? [
+ escaped?: no
+ ]
+ in-comment? [
+ switch c [
+ #"^/" [in-comment?: no]
+ ]
+ ]
+ 'else [
+ switch c [
+ #"^^" [escaped?: yes]
+ #";" [if zero? list/2 [in-comment?: yes]]
+ #"[" [list/1: list/1 + 1]
+ #"]" [list/1: list/1 - 1]
+ #"{" [list/2: list/2 + 1]
+ #"}" [list/2: list/2 - 1]
+ ]
+ ]
+ ]
+ ]
+ list
+]
+
+do-console: function [][
+ buffer: make string! 10000
+ prompt: red-prompt: "red>> "
+ mode: 'mono
+
+ switch-mode: [
+ mode: case [
+ cnt/1 > 0 ['block]
+ cnt/2 > 0 ['string]
+ 'else [
+ prompt: red-prompt
+ do eval
+ 'mono
+ ]
+ ]
+ prompt: switch mode [
+ block ["[^-"]
+ string ["{^-"]
+ mono [red-prompt]
+ ]
+ ]
+
+ eval: [
+ code: load/all buffer
+
+ unless tail? code [
+ set/any 'result do code
+
+ unless unset? :result [
+ if 67 = length? result: mold/part :result 67 [ ;-- optimized for width = 72
+ clear back tail result
+ append result "..."
+ ]
+ print ["==" result]
+ ]
+ ]
+ clear buffer
+ ]
+
+ while [true][
+ unless tail? line: input prompt [
+ append buffer line
+ cnt: count-delimiters buffer
+
+ either Windows? [
+ remove skip tail buffer -2 ;-- clear extra CR (Windows)
+ ][
+ append buffer lf ;-- Unix
+ ]
+
+ switch mode [
+ block [if cnt/1 <= 0 [do switch-mode]]
+ string [if cnt/2 <= 0 [do switch-mode]]
+ mono [do either any [cnt/1 > 0 cnt/2 > 0][switch-mode][eval]]
+ ]
+ ]
+ ]
+]
+
+q: :quit
+
+if script: read-argument [
+ script: load script
+ either any [
+ script/1 <> 'Red
+ not block? script/2
+ ][
+ print "*** Error: not a Red program!"
+ ][
+ do skip script 2
+ ]
+ quit
+]
+
+init-console "Red Console"
+
+print {
+-=== Red Console alpha version ===-
+(only ASCII input supported)
+}
+
+do-console
\ No newline at end of file diff --git a/tests/examplefiles/example.reds b/tests/examplefiles/example.reds new file mode 100644 index 00000000..eb92310d --- /dev/null +++ b/tests/examplefiles/example.reds @@ -0,0 +1,150 @@ +Red/System [ + Title: "Red/System example file" + Purpose: "Just some code for testing Pygments colorizer" + Language: http://www.red-lang.org/ +] + +#include %../common/FPU-configuration.reds + +; C types + +#define time! long! +#define clock! long! + +date!: alias struct! [ + second [integer!] ; 0-61 (60?) + minute [integer!] ; 0-59 + hour [integer!] ; 0-23 + + day [integer!] ; 1-31 + month [integer!] ; 0-11 + year [integer!] ; Since 1900 + + weekday [integer!] ; 0-6 since Sunday + yearday [integer!] ; 0-365 + daylight-saving-time? [integer!] ; Negative: unknown +] + +#either OS = 'Windows [ + #define clocks-per-second 1000 +][ + ; CLOCKS_PER_SEC value for Syllable, Linux (XSI-conformant systems) + ; TODO: check for other systems + #define clocks-per-second 1000'000 +] + +#import [LIBC-file cdecl [ + + ; Error handling + + form-error: "strerror" [ ; Return error description. + code [integer!] + return: [c-string!] + ] + print-error: "perror" [ ; Print error to standard error output. + string [c-string!] + ] + + + ; Memory management + + make: "calloc" [ ; Allocate zero-filled memory. + chunks [size!] + size [size!] + return: [binary!] + ] + resize: "realloc" [ ; Resize memory allocation. + memory [binary!] + size [size!] + return: [binary!] + ] + ] + + JVM!: alias struct! [ + reserved0 [int-ptr!] + reserved1 [int-ptr!] + reserved2 [int-ptr!] + + DestroyJavaVM [function! [[JNICALL] vm [JVM-ptr!] return: [jint!]]] + AttachCurrentThread [function! [[JNICALL] vm [JVM-ptr!] penv [struct! [p [int-ptr!]]] args [byte-ptr!] return: [jint!]]] + DetachCurrentThread [function! [[JNICALL] vm [JVM-ptr!] return: [jint!]]] + GetEnv [function! [[JNICALL] vm [JVM-ptr!] penv [struct! [p [int-ptr!]]] version [integer!] return: [jint!]]] + AttachCurrentThreadAsDaemon [function! [[JNICALL] vm [JVM-ptr!] penv [struct! [p [int-ptr!]]] args [byte-ptr!] return: [jint!]]] +] + + ;just some datatypes for testing: + + #some-hash + 10-1-2013 + quit + + ;binary: + #{00FF0000} + #{00FF0000 FF000000} + #{00FF0000 FF000000} ;with tab instead of space + 2#{00001111} + 64#{/wAAAA==} + 64#{/wAAA A==} ;with space inside + 64#{/wAAA A==} ;with tab inside + + + ;string with char + {bla ^(ff) foo} + {bla ^(( foo} + ;some numbers: + 12 + 1'000 + 1.2 + FF00FF00h + + ;some tests of hexa number notation with not common ending + [ff00h ff00h] ff00h{} FFh"foo" 00h(1 + 2) (AEh) + +;normal words: +foo char + +;get-word +:foo + +;lit-word: +'foo 'foo + +;multiple comment tests... +1 + 1 +comment "aa" +2 + 2 +comment {aa} +3 + 3 +comment {a^{} +4 + 4 +comment {{}} +5 + 5 +comment { + foo: 6 +} +6 + 6 +comment [foo: 6] +7 + 7 +comment [foo: "[" ] +8 + 8 +comment [foo: {^{} ] +9 + 9 +comment [foo: {boo} ] +10 + 10 +comment 5-May-2014/11:17:34+2:00 +11 + 11 + + +to-integer foo +foo/(a + 1)/b + +call/output reform ['which interpreter] path: copy "" + + version-1.1: 00010001h + + #if type = 'exe [ + push system/stack/frame ;-- save previous frame pointer + system/stack/frame: system/stack/top ;-- @@ reposition frame pointer just after the catch flag +] +push CATCH_ALL ;-- exceptions root barrier +push 0 ;-- keep stack aligned on 64-bit
\ No newline at end of file diff --git a/tests/examplefiles/example.rkt b/tests/examplefiles/example.rkt index a3e4a29e..acc0328e 100644 --- a/tests/examplefiles/example.rkt +++ b/tests/examplefiles/example.rkt @@ -1,5 +1,7 @@ #lang racket +(require (only-in srfi/13 string-contains)) + ; Single-line comment style. ;; Single-line comment style. @@ -8,45 +10,259 @@ #| Multi-line comment style ... +#|### #| nested |#||| |# ... on multiple lines |# -(define (a-function x #:keyword [y 0]) +#;(s-expression comment (one line)) + +#; +(s-expression comment + (multiple lines)) + +#! shebang comment + +#!/shebang comment + +#! shebang \ +comment + +#!/shebang \ +comment + +;; Uncommented numbers after single-line comments +;; NEL
133 +;; LS
8232 +;; PS
8233 + +#reader racket +(define(a-function x #:keyword [y 0]) (define foo0 'symbol) ; () [define foo1 'symbol] ; [] {define foo2 'symbol} ; {} - (and (append (car '(1 2 3)))) + (define 100-Continue 'symbol) + (and (append (car'(1 2 3)))) (regexp-match? #rx"foobar" "foobar") - (regexp-match? #px"foobar" "foobar") - (define a 1)) - (let ([b "foo"]) - (displayln b)) + (regexp-match? #px"\"foo\\(bar\\)?\"" "foobar") + (regexp-match? #rx#"foobar" "foobar") + (regexp-match? #px#"foobar" "foobar") + (define #csa 1) + #Ci (let ([#%A|||b #true C +\|ｄ "foo"]) + (displayln #cS #%\ab\ #true\ C\ +\\ｄ||)) (for/list ([x (in-list (list 1 2 (list 3 4)))]) - (cond - [(pair? x) (car x)] - [else x]))) + (cond + [(pair? x) (car x)] + [else x]))) -;; Literal number examples +;; Literals (values ;; #b - #b1.1 - #b-1.1 - #b1e1 - #b0/1 - #b1/1 - #b1e-1 - #b101 - + #b1 + #b+1 + #b-1 + #b.1 + #b1. + #b0.1 + #b+0.1 + #b-0.1 + #b1/10 + #b+1/10 + #b-1/10 + #b1e11 + #b+1e11 + #b-1e11 + #b.1e11 + #b1.e11 + #b0.1e11 + #b+0.1e11 + #b-0.1e11 + #b1/10e11 + #b+1/10e11 + #b-1/10e11 + #b+i + #b1+i + #b+1+i + #b-1+i + #b.1+i + #b1.+i + #b0.1+i + #b+0.1+i + #b-0.1+i + #b1/10+i + #b+1/10+i + #b-1/10+i + #b1e11+i + #b+1e11+i + #b-1e11+i + #b1.e11+i + #b.1e11+i + #b0.1e11+i + #b+0.1e11+i + #b-0.1e11+i + #b1/10e11+i + #b+1/10e11+i + #b-1/10e11+i + #b+1i + #b1+1i + #b+1+1i + #b-1+1i + #b1.+1i + #b.1+1i + #b0.1+1i + #b+0.1+1i + #b-0.1+1i + #b1/10+1i + #b+1/10+1i + #b-1/10+1i + #b1e11+1i + #b+1e11+1i + #b-1e11+1i + #b.1e11+1i + #b0.1e11+1i + #b+0.1e11+1i + #b-0.1e11+1i + #b1/10e11+1i + #b+1/10e11+1i + #b-1/10e11+1i + #b+1/10e11i + #b1+1/10e11i + #b+1+1/10e11i + #b-1+1/10e11i + #b.1+1/10e11i + #b0.1+1/10e11i + #b+0.1+1/10e11i + #b-0.1+1/10e11i + #b1/10+1/10e11i + #b+1/10+1/10e11i + #b-1/10+1/10e11i + #b1e11+1/10e11i + #b+1e11+1/10e11i + #b-1e11+1/10e11i + #b.1e11+1/10e11i + #b0.1e11+1/10e11i + #b+0.1e11+1/10e11i + #b-0.1e11+1/10e11i + #b1/10e11+1/10e11i + #b+1/10e11+1/10e11i + #b-1/10e11+1/10e11i ;; #d - #d-1.23 - #d1.123 - #d1e3 - #d1e-22 - #d1/2 - #d-1/2 #d1 + #d+1 #d-1 - + #d.1 + #d1. + #d1.2 + #d+1.2 + #d-1.2 + #d1/2 + #d+1/2 + #d-1/2 + #d1e3 + #d+1e3 + #d-1e3 + #d.1e3 + #d1.e3 + #d1.2e3 + #d+1.2e3 + #d-1.2e3 + #d1/2e3 + #d+1/2e3 + #d-1/2e3 + #d+i + #d1+i + #d+1+i + #d-1+i + #d.1+i + #d1.+i + #d1.2+i + #d+1.2+i + #d-1.2+i + #d1/2+i + #d+1/2+i + #d-1/2+i + #d1e3+i + #d+1e3+i + #d-1e3+i + #d1.e3+i + #d.1e3+i + #d1.2e3+i + #d+1.2e3+i + #d-1.2e3+i + #d1/2e3+i + #d+1/2e3+i + #d-1/2e3+i + #d+1i + #d1+1i + #d+1+1i + #d-1+1i + #d1.+1i + #d.1+1i + #d1.2+1i + #d+1.2+1i + #d-1.2+1i + #d1/2+1i + #d+1/2+1i + #d-1/2+1i + #d1e3+1i + #d+1e3+1i + #d-1e3+1i + #d.1e3+1i + #d1.2e3+1i + #d+1.2e3+1i + #d-1.2e3+1i + #d1/2e3+1i + #d+1/2e3+1i + #d-1/2e3+1i + #d+1/2e3i + #d1+1/2e3i + #d+1+1/2e3i + #d-1+1/2e3i + #d.1+1/2e3i + #d1.2+1/2e3i + #d+1.2+1/2e3i + #d-1.2+1/2e3i + #d1/2+1/2e3i + #d+1/2+1/2e3i + #d-1/2+1/2e3i + #d1e3+1/2e3i + #d+1e3+1/2e3i + #d-1e3+1/2e3i + #d.1e3+1/2e3i + #d1.2e3+1/2e3i + #d+1.2e3+1/2e3i + #d-1.2e3+1/2e3i + #d1/2e3+1/2e3i + #d+1/2e3+1/2e3i + #d-1/2e3+1/2e3i + ;; Extflonums + +nan.t + 1t3 + +1t3 + -1t3 + .1t3 + 1.t3 + 1.2t3 + +1.2t3 + -1.2t3 + 1/2t3 + +1/2t3 + -1/2t3 + 1#t0 + 1.#t0 + .2#t0 + 1.2#t0 + 1#/2t0 + 1/2#t0 + 1#/2#t0 + 1#t3 + 1.#t3 + .2#t3 + 1.2#t3 + 1#/2t3 + 1/2#t3 + 1#/2#t3 ;; No # reader prefix -- same as #d -1.23 1.123 @@ -56,7 +272,6 @@ Multi-line comment style ... -1/2 1 -1 - ;; #e #e-1.23 #e1.123 @@ -66,7 +281,24 @@ Multi-line comment style ... #e-1 #e1/2 #e-1/2 - + ;; #d#e + #d#e-1.23 + #d#e1.123 + #d#e1e3 + #d#e1e-22 + #d#e1 + #d#e-1 + #d#e1/2 + #d#e-1/2 + ;; #e#d + #e#d-1.23 + #e#d1.123 + #e#d1e3 + #e#d1e-22 + #e#d1 + #e#d-1 + #e#d1/2 + #e#d-1/2 ;; #i always float #i-1.23 #i1.123 @@ -76,7 +308,126 @@ Multi-line comment style ... #i-1/2 #i1 #i-1 - + ;; Implicitly inexact numbers + +nan.0 + 1# + 1.# + .2# + 1.2# + 1#/2 + 1/2# + 1#/2# + 1#e3 + 1.#e3 + .2#e3 + 1.2#e3 + 1#/2e3 + 1/2#e3 + 1#/2#e3 + +nan.0+i + 1#+i + 1.#+i + .2#+i + 1.2#+i + 1#/2+i + 1/2#+i + 1#/2#+i + 1#e3+i + 1.#e3+i + .2#e3+i + 1.2#e3+i + 1#/2e3+i + 1/2#e3+i + 1#/2#e3+i + +nan.0i + +1#i + +1.#i + +.2#i + +1.2#i + +1#/2i + +1/2#i + +1#/2#i + +1#e3i + +1.#e3i + +.2#e3i + +1.2#e3i + +1#/2e3i + +1/2#e3i + +1#/2#e3i + 0+nan.0i + 0+1#i + 0+1.#i + 0+.2#i + 0+1.2#i + 0+1#/2i + 0+1/2#i + 0+1#/2#i + 0+1#e3i + 0+1.#e3i + 0+.2#e3i + 0+1.2#e3i + 0+1#/2e3i + 0+1/2#e3i + 0+1#/2#e3i + 1#/2#e3+nan.0i + 1#/2#e3+1#i + 1#/2#e3+1.#i + 1#/2#e3+.2#i + 1#/2#e3+1.2#i + 1#/2#e3+1#/2i + 1#/2#e3+1/2#i + 1#/2#e3+1#/2#i + 1#/2#e3+1#e3i + 1#/2#e3+1.#e3i + 1#/2#e3+.2#e3i + 1#/2#e3+1.2#e3i + 1#/2#e3+1#/2e3i + 1#/2#e3+1/2#e3i + 1#/2#e3+1#/2#e3i + +nan.0@1 + 1#@1 + 1.#@1 + .2#@1 + 1.2#@1 + 1#/2@1 + 1/2#@1 + 1#/2#@1 + 1#e3@1 + 1.#e3@1 + .2#e3@1 + 1.2#e3@1 + 1#/2e3@1 + 1/2#e3@1 + 1#/2#e3@1 + 1@+nan.0 + 1@1# + 1@1.# + 1@.2# + 1@1.2# + 1@1#/2 + 1@1/2# + 1@1#/2# + 1@1#e3 + 1@1.#e3 + 1@.2#e3 + 1@1.2#e3 + 1@1#/2e3 + 1@1/2#e3 + 1@1#/2#e3 + 1#/2#e3@1# + 1#/2#e3@1.# + 1#/2#e3@.2# + 1#/2#e3@1.2# + 1#/2#e3@1#/2 + 1#/2#e3@1/2# + 1#/2#e3@1#/2# + 1#/2#e3@1#e3 + 1#/2#e3@1.#e3 + 1#/2#e3@.2#e3 + 1#/2#e3@1.2#e3 + 1#/2#e3@1#/2e3 + 1#/2#e3@1/2#e3 + 1#/2#e3@1#/2#e3 ;; #o #o777.777 #o-777.777 @@ -86,10 +437,307 @@ Multi-line comment style ... #o-3/7 #o777 #o-777 - + #e#o777.777 + #e#o-777.777 + #e#o777e777 + #e#o777e-777 + #e#o3/7 + #e#o-3/7 + #e#o777 + #e#o-777 + #i#o777.777 + #i#o-777.777 + #i#o777e777 + #i#o777e-777 + #i#o3/7 + #i#o-3/7 + #i#o777 + #i#o-777 ;; #x #x-f.f #xf.f + #xfsf + #xfs-f + #x7/f + #x-7/f #x-f #xf + #e#x-f.f + #e#xf.f + #e#xfsf + #e#xfs-f + #e#x7/f + #e#x-7/f + #e#x-f + #e#xf + #i#x-f.f + #i#xf.f + #i#xfsf + #i#xfs-f + #i#x7/f + #i#x-7/f + #i#x-f + #i#xf + ;; Not numbers + '-1.23x + '1.123x + '1e3x + '1e-22x + '1/2x + '-1/2x + '1x + '-1x + '/ + '1/ + '/2 + '1//2 + '1e3. + '1e + 'e3 + '.i + '1.2.3 + '1..2 + '.1. + '@ + '1@ + '@2 + '1@@2 + '1@2@3 + '1@2i + '1+-2i + '1i+2 + '1i+2i + '1+2i+3i + '- + '--1 + '+ + '++1 + '1/2.3 + '1#2 + '1#.2 + '1.#2 + '.#2 + '+nan.t+nan.ti + '+nan.t@nan.t + ;; Booleans + #t + #T + #true + #f + #F + #false + ;; Characters, strings, and byte strings + #\ + #\Null9 + #\n9 + #\99 + #\0009 + #\u3BB + #\u03BB9 + #\U3BB + #\U000003BB9 + #\λ9 + "string\ + \a.\b.\t.\n.\v.\f.\r.\e.\".\'.\\.\1.\123.\1234.\x9.\x30.\x303" + "\u9.\u1234.\u12345.\U9.\U00100000.\U001000000" + #"byte-string\7\xff\t" + #<<HERE STRING +lorem ipsum +dolor sit amet +consectetur HERE STRING +HERE STRING adipisicing elit +HERE STRING + #| +HERE STRING +|# + ;; Other literals + #(vector) + #20() + #s[prefab-structure 1 2 3] + #&{box} + #hash(("a" . 5)) + #hasheq((a . 5) (b . 7)) + #hasheqv((a . 5) (b . 7)) + #'(define x 1) + #`(define x #,pi) + ;; quote, quasiquote, and unquote + 'pi + ' pi + ''pi + '`pi + '`,pi + ',pi + `pi + ` pi + `'pi + ``pi + `,pi + ` , pi + `,'pi + `,`pi + `,`,pi + '(+) + ' (+) + ''(+) + '`(+) + ',(+) + `(+) + ` (+) + `'(+) + ``(+) + `,(+) + ` , (+) + `,'(+) + `,`(+) + `,`,(+) + #readerracket/base'pi.f + '#readerracket/base pi.f + #readerracket/base`pi.f + `#readerracket/base pi.f + #readerracket/base`,pi.f + `#readerracket/base,pi.f + `,#readerracket/base pi.f + #readerracket/base'`,pi.f + '#readerracket/base`,pi.f + '`#readerracket/base,pi.f + '`,#readerracket/base pi.f + #readerracket/base'(*) + '#readerracket/base(*) + #readerracket/base`(*) + `#readerracket/base(*) + #readerracket/base`,(*) + `#readerracket/base,(*) + `,#readerracket/base(*) + #readerracket/base'`,(*) + '#readerracket/base`,(*) + '`#readerracket/base,(*) + '`,#readerracket/base(*) + (quote pi) + (quote (quote pi)) + (quote (quasiquote pi)) + (quote (quasiquote (unquote pi))) + (quote (unquote pi)) + (quasiquote pi) + (quasiquote (quote pi)) + (quasiquote (quasiquote pi)) + (quasiquote (unquote pi)) + (quasiquote (unquote (quote pi))) + (quasiquote (unquote (quasiquote pi))) + (quasiquote (unquote (quasiquote (unquote pi)))) + (quote (+)) + (quote (quote (+))) + (quote (quasiquote (+))) + (quote (unquote (+))) + (quasiquote (+)) + (quasiquote (quote (+))) + (quasiquote (quasiquote (+))) + (quasiquote (unquote (+))) + (quasiquote (unquote (quote (+)))) + (quasiquote (unquote (quasiquote (+)))) + (quasiquote (unquote (quasiquote (unquote (+))))) + #reader racket/base (quote pi.f) + (quote #reader racket/base pi.f) + #reader racket/base (quasiquote pi.f) + (quasiquote #reader racket/base pi.f) + #reader racket/base (quasiquote (unquote pi.f)) + (quasiquote #reader racket/base (unquote pi.f)) + (quasiquote (unquote #reader racket/base pi.f)) + #reader racket/base (quote (quasiquote (unquote pi.f))) + (quote #reader racket/base (quasiquote (unquote pi.f))) + (quote (quasiquote #reader racket/base (unquote pi.f))) + (quote (quasiquote (unquote #reader racket/base pi.f))) + #reader racket/base (quote (*)) + (quote #reader racket/base (*)) + #reader racket/base (quasiquote (*)) + (quasiquote #reader racket/base (*)) + #reader racket/base (quasiquote (unquote (*))) + (quasiquote #reader racket/base (unquote (*))) + (quasiquote (unquote #reader racket/base (*))) + #reader racket/base (quote (quasiquote (unquote (*)))) + (quote #reader racket/base (quasiquote (unquote (*)))) + (quote (quasiquote #reader racket/base (unquote (*)))) + (quote (quasiquote (unquote #reader racket/base (*)))) + ;; Make sure non-identifiers work with quotes + ' "" pi + ' #t pi + ' #() pi + ' #s(s) pi + ' #\u3BB pi + ' #\U000003BB pi + ' #\space pi + ' #\. pi + ' #"" pi + ' #:kw pi + ' #&b pi + ' #'(define x 1) pi + ' #`(define x #,pi) pi + ' #I0 pi + ' #E0 pi + ' #X0 pi + ' #O0 pi + ' #D0 pi + ' #B0 pi + ' #<<EOF +EOF + pi + ' #rx"" pi + ' #rx#"" pi + ' #px"" pi + ' #px#"" pi + ' #hash() pi + ' #hasheq[] pi + ' #hasheqv{} pi + ' #1(v) pi ) + +;; Use the following to generate lists of built-ins and keywords. +;; Run +;; (displayln (wrap-lines KEYWORDS)) +;; (displayln (wrap-lines BUILTINS)) +;; and copy the results into RacketLexer._keywords and RacketLexer._builtins. + +;; (-> (listof string?) string?) +;; Appends all the strings together, quoting them as appropriate for Python, +;; with commas and spaces between them, wrapping at 80 characters, with an +;; indentation of 8 spaces. +(define (wrap-lines lst) + (define INDENTATION '" ") + (define WIDTH '80) + (define (wrap-lines* lst done-lines current-line) + (if (null? lst) + (string-append (foldr string-append "" done-lines) current-line) + (let* ([str (first lst)] + [wrapped-str (if (regexp-match-exact? '#px"[[:ascii:]]+" str) + (string-append "'" str "',") + (string-append "u'" str "',"))] + [new-line (string-append current-line " " wrapped-str)]) + (if ((string-length new-line) . >= . WIDTH) + (wrap-lines* (rest lst) + (append done-lines + `(,(string-append current-line "\n"))) + (string-append INDENTATION wrapped-str)) + (wrap-lines* (rest lst) + done-lines + new-line))))) + (wrap-lines* lst '() INDENTATION)) + +;; (-> string? boolean?) +;; Returns #t if str represents a syntax identifier in the current namespace, +;; otherwise #f. +(define (syntax-identifier? str) + (with-handlers ([exn? exn?]) + (not (eval (call-with-input-string str read))))) + +(define RACKET-NAMESPACE + (parameterize ([current-namespace (make-base-namespace)]) + (namespace-require 'racket) + (current-namespace))) + +(define BOUND-IDENTIFIERS + (parameterize ([current-namespace RACKET-NAMESPACE]) + (sort (map symbol->string (namespace-mapped-symbols)) + string<=?))) + +(define-values (KEYWORDS BUILTINS) + (parameterize ([current-namespace RACKET-NAMESPACE]) + (partition syntax-identifier? BOUND-IDENTIFIERS))) diff --git a/tests/examplefiles/example.sh b/tests/examplefiles/example.sh new file mode 100644 index 00000000..2112cdd1 --- /dev/null +++ b/tests/examplefiles/example.sh @@ -0,0 +1,22 @@ +#!/bin/bash + +printf "%d %s\n" 10 "foo" +printf "%d %s\n" $((10#1)) "bar" + +let "m = 10#${1:1:2}" +echo $m + +m=$((10#${1:4:3} + 10#${1:1:3})) +echo $m + +m=$((10#${1:4:3})) +echo $m + +m=$((10#$1)) +echo $m + +m=$((10#1)) +echo $m + +m=$((10)) +echo $m diff --git a/tests/examplefiles/import.hs b/tests/examplefiles/import.hs deleted file mode 100644 index 09058ae6..00000000 --- a/tests/examplefiles/import.hs +++ /dev/null @@ -1,4 +0,0 @@ -import "mtl" Control.Monad.Trans - -main :: IO () -main = putStrLn "hello world" diff --git a/tests/examplefiles/livescript-demo.ls b/tests/examplefiles/livescript-demo.ls index 16d1894a..03cbcc99 100644 --- a/tests/examplefiles/livescript-demo.ls +++ b/tests/examplefiles/livescript-demo.ls @@ -7,7 +7,7 @@ dashes-identifiers = -> underscores_i$d = -> /regexp1/ //regexp2//g - 'strings' and "strings" and \strings + 'strings' and "strings" and \strings and \#$-"\'strings another-word-list = <[ more words ]> diff --git a/tests/examplefiles/main.cmake b/tests/examplefiles/main.cmake index dac3da43..71dc3ce7 100644 --- a/tests/examplefiles/main.cmake +++ b/tests/examplefiles/main.cmake @@ -1,3 +1,5 @@ +CMAKE_MINIMUM_REQUIRED(VERSION 2.6 FATAL_ERROR) + SET( SOURCES back.c io.c main.c ) MESSAGE( ${SOURCES} ) # three arguments, prints "back.cio.cmain.c" MESSAGE( "${SOURCES}" ) # one argument, prints "back.c;io.c;main.c" diff --git a/tests/examplefiles/objc_example.m b/tests/examplefiles/objc_example.m index f4f27170..f3f85f65 100644 --- a/tests/examplefiles/objc_example.m +++ b/tests/examplefiles/objc_example.m @@ -1,35 +1,179 @@ -#import "Somefile.h" +// Test various types of includes +#import <Foundation/Foundation.h> +# import <AppKit/AppKit.h> +#import "stdio.h" +#\ + import \ + "stdlib.h" +# /*line1*/ \ +import /* line 2 */ \ +"stdlib.h" // line 3 -@implementation ABC +// Commented out code with preprocessor +#if 0 +#define MY_NUMBER 3 +#endif -- (id)a:(B)b { - return 1; + #\ + if 1 +#define TEST_NUMBER 3 +#endif + +// Empty preprocessor +# + +// Class forward declaration +@class MyClass; + +// Empty classes +@interface EmptyClass +@end +@interface EmptyClass2 +{ +} +@end +@interface EmptyClass3 : EmptyClass2 +{ +} +@end + +// Custom class inheriting from built-in +@interface MyClass : NSObject +{ +@public + NSString *myString; + __weak NSString *_weakString; +@protected + NSTextField *_textField; +@private + NSDate *privateDate; } +// Various property aatributes +@property(copy, readwrite, nonatomic) NSString *myString; +@property(weak) NSString *weakString; +@property(retain, strong, atomic) IBOutlet NSTextField *textField; + +// Class methods ++ (void)classMethod1:(NSString *)arg; ++ (void)classMethod2:(NSString *) arg; // Test space before arg + @end -@implementation ABC +typedef id B; -- (void)xyz; +#pragma mark MyMarker +// MyClass.m +// Class extension to declare private property +@interface MyClass () +@property(retain) NSDate *privateDate; +- (void)hiddenMethod; @end -NSDictionary *dictionary = [NSDictionary dictionaryWithObjectsAndKeys: - @"quattuor", @"four", @"quinque", @"five", @"sex", @"six", nil]; +// Special category +@interface MyClass (Special) +@property(retain) NSDate *specialDate; +@end +@implementation MyClass +@synthesize myString; +@synthesize privateDate; -NSString *key; -for (key in dictionary) { - NSLog(@"English: %@, Latin: %@", key, [dictionary valueForKey:key]); -} +- (id)a:(B)b { + /** + * C-style comment + */ + + // Selector keywords/types + SEL someMethod = @selector(hiddenMethod); + + // Boolean types + Boolean b1 = FALSE; + BOOL b2 = NO; + bool b3 = true; + + /** + * Number literals + */ + // Int Literal + NSNumber *n1 = @( 1 ); + // Method call + NSNumber *n2 = @( [b length] ); + // Define variable + NSNumber *n3 = @( TEST_NUMBER ); + // Arthimetic expression + NSNumber *n4 = @(1 + 2); + // From variable + int myInt = 5; + NSNumber *n5 = @(myInt); + // Nest expression + NSNumber *n6 = @(1 + (2 + 6.0)); + // Bool literal + NSNumber *n7 = @NO; + // Bool expression + NSNumber *n8 = @(YES); + // Character + NSNumber *n9 = @'a'; + // int + NSNumber *n10 = @123; + // unsigned + NSNumber *n11 = @1234U; + // long + NSNumber *n12 = @1234567890L; + // float + NSNumber *n13 = @3.14F; + // double + NSNumber *n14 = @3.14F; + + // Array literals + NSArray *arr = @[ @"1", @"2" ]; + arr = @[ @[ @"1", @"2" ], [arr lastObject] ]; + [arr lastObject]; + [@[ @"1", @"2" ] lastObject]; + + // Dictionary literals + NSDictionary *d = @{ @"key": @"value" }; + [[d allKeys] lastObject]; + [[@{ @"key": @"value" } allKeys] lastObject]; + d = @{ @"key": @{ @"key": @"value" } }; + + [self hiddenMethod]; + [b length]; + [privateDate class]; -// Literals -NSArray *a = @[ @"1", @"2" ]; + NSDictionary *dictionary = [NSDictionary dictionaryWithObjectsAndKeys: + @"1", @"one", @"2", @"two", @"3", @"three", nil]; + + NSString *key; + for (key in dictionary) { + NSLog(@"Number: %@, Word: %@", key, [dictionary valueForKey:key]); + } -NSDictionary *d = @{ @"key": @"value" }; + // Blocks + int (^myBlock)(int arg1, int arg2); + NSString *(^myName)(NSString *) = ^(NSString *value) { + return value; + }; -NSNumber *n1 = @( 1 ); -NSNumber *n2 = @( [a length] ); + return nil; +} + +- (void)hiddenMethod { + // Synchronized block + @synchronized(self) { + [myString retain]; + [myString release]; + } +} -+ (void)f1:(NSString *)s1; -+ (void)f2:(NSString *) s2; ++ (void)classMethod1:(NSString *)arg {} ++ (void)classMethod2:(NSString *) arg +{ + // Autorelease pool block + @autoreleasepool { + NSLog(@"Hello, World!"); + } +} + +@end diff --git a/tests/examplefiles/objc_example2.m b/tests/examplefiles/objc_example2.m deleted file mode 100644 index b7a5a685..00000000 --- a/tests/examplefiles/objc_example2.m +++ /dev/null @@ -1,27 +0,0 @@ -// MyClass.h -@interface MyClass : NSObject -{ - NSString *value; - NSTextField *textField; -@private - NSDate *lastModifiedDate; -} -@property(copy, readwrite) NSString *value; -@property(retain) IBOutlet NSTextField *textField; -@end - -// MyClass.m -// Class extension to declare private property -@interface MyClass () -@property(retain) NSDate *lastModifiedDate; -@end - -@implementation MyClass -@synthesize value; -@synthesize textField; -@synthesize lastModifiedDate; -// implementation continues -@end - -+ (void)f1:(NSString *)s1; -+ (void)f2:(NSString *)s2; diff --git a/tests/examplefiles/qbasic_example b/tests/examplefiles/qbasic_example new file mode 100644 index 00000000..27041af6 --- /dev/null +++ b/tests/examplefiles/qbasic_example @@ -0,0 +1,2 @@ +10 print RIGHT$("hi there", 5) +20 goto 10 diff --git a/tests/examplefiles/test.cyp b/tests/examplefiles/test.cyp new file mode 100644 index 00000000..37465a4d --- /dev/null +++ b/tests/examplefiles/test.cyp @@ -0,0 +1,123 @@ +//test comment +START a = node(*) +MATCH (a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) +RETURN a.name, m.title, d.name; + +START a = node(*) +MATCH (a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) +WITH d,m,count(a) as Actors +WHERE Actors > 4 +RETURN d.name as Director,m.title as Movie, Actors ORDER BY Actors; + +START a=node(*) +MATCH p=(a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) +return p; + +START a = node(*) +MATCH p1=(a)-[:ACTED_IN]->(m), p2=d-[:DIRECTED]->(m) +WHERE m.title="The Matrix" +RETURN p1, p2; + +START a = node(*) +MATCH (a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) +WHERE a=d +RETURN a.name; + +START a = node(*) +MATCH (a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) +WHERE a=d +RETURN a.name; + +START a=node(*) +MATCH (a)-[:ACTED_IN]->(m)<-[:DIRECTED]-(d) +RETURN a.name, d.name, count(*) as Movies,collect(m.title) as Titles +ORDER BY (Movies) DESC +LIMIT 5; + +START keanu=node:node_auto_index(name="Keanu Reeves") +RETURN keanu; + +START keanu=node:node_auto_index(name="Keanu Reeves") +MATCH (keanu)-[:ACTED_IN]->(movie) +RETURN movie.title; + +START keanu=node:node_auto_index(name="Keanu Reeves") +MATCH (keanu)-[r:ACTED_IN]->(movie) +WHERE "Neo" in r.roles +RETURN DISTINCT movie.title; + +START keanu=node:node_auto_index(name="Keanu Reeves") +MATCH (keanu)-[:ACTED_IN]->()<-[:DIRECTED]-(director) +RETURN director.name; + +START keanu=node:node_auto_index(name="Keanu Reeves") +MATCH (keanu)-[:ACTED_IN]->(movie)<-[:ACTED_IN]-(n) +WHERE n.born < keanu.born +RETURN DISTINCT n.name, keanu.born ,n.born; + +START keanu=node:node_auto_index(name="Keanu Reeves"), + hugo=node:node_auto_index(name="Hugo Weaving") +MATCH (keanu)-[:ACTED_IN]->(movie) +WHERE NOT((hugo)-[:ACTED_IN]->(movie)) +RETURN DISTINCT movie.title; + +START a = node(*) +MATCH (a)-[:ACTED_IN]->(m) +WITH a,count(m) as Movies +RETURN a.name as Actor, Movies ORDER BY Movies; + +START keanu=node:node_auto_index(name="Keanu Reeves"),actor +MATCH past=(keanu)-[:ACTED_IN]->()<-[:ACTED_IN]-(), + actors=(actor)-[:ACTED_IN]->() +WHERE hasnt=actors NOT IN past +RETURN hasnt; + +START keanu=node:node_auto_index(name="Keanu Reeves") +MATCH (keanu)-[:ACTED_IN]->()<-[:ACTED_IN]-(c), + (c)-[:ACTED_IN]->()<-[:ACTED_IN]-(coc) +WHERE NOT((keanu)-[:ACTED_IN]->()<-[:ACTED_IN]-(coc)) +AND coc > keanu +RETURN coc.name, count(coc) +ORDER BY count(coc) DESC +LIMIT 3; + +START kevin=node:node_auto_index(name="Kevin Bacon"), + movie=node:node_auto_index(name="Mystic River") +MATCH (kevin)-[:ACTED_IN]->(movie) +RETURN DISTINCT movie.title; + +CREATE (n + { + title:"Mystic River", + released:1993, + tagline:"We bury our sins here, Dave. We wash them clean." + } + ) RETURN n; + + +START movie=node:node_auto_index(title="Mystic River") +SET movie.released = 2003 +RETURN movie; + +start emil=node:node_auto_index(name="Emil Eifrem") MATCH emil-[r]->(n) DELETE r, emil; + +START a=node(*) +MATCH (a)-[:ACTED_IN]->()<-[:ACTED_IN]-(b) +CREATE UNIQUE (a)-[:KNOWS]->(b); + +START keanu=node:node_auto_index(name="Keanu Reeves") +MATCH (keanu)-[:KNOWS*2]->(fof) +WHERE keanu <> fof +RETURN distinct fof.name; + +START charlize=node:node_auto_index(name="Charlize Theron"), + bacon=node:node_auto_index(name="Kevin Bacon") +MATCH p=shortestPath((charlize)-[:KNOWS*]->(bacon)) +RETURN extract(n in nodes(p) | n.name)[1]; + +START actors=node: + +MATCH (alice)-[:`REALLY LIKES`]->(bob) +MATCH (alice)-[:`REALLY ``LIKES```]->(bob) +myFancyIdentifier.`(weird property name)` +"string\t\n\b\f\\\''\"" diff --git a/tests/examplefiles/test.pan b/tests/examplefiles/test.pan new file mode 100644 index 00000000..56c8bd62 --- /dev/null +++ b/tests/examplefiles/test.pan @@ -0,0 +1,54 @@ +object template pantest; + +# Very simple pan test file +"/long/decimal" = 123; +"/long/octal" = 0755; +"/long/hexadecimal" = 0xFF; + +"/double/simple" = 0.01; +"/double/pi" = 3.14159; +"/double/exponent" = 1e-8; +"/double/scientific" = 1.3E10; + +"/string/single" = 'Faster, but escapes like \t, \n and \x3d don''t work, but '' should work.'; +"/string/double" = "Slower, but escapes like \t, \n and \x3d do work"; + +variable TEST = 2; + +"/x2" = to_string(TEST); +"/x2" ?= 'Default value'; + +"/x3" = 1 + 2 + value("/long/decimal"); + +"/x4" = undef; + +"/x5" = null; + +variable e ?= error("Test error message"); + +# include gmond config for services-monitoring +include { 'site/ganglia/gmond/services-monitoring' }; + +"/software/packages"=pkg_repl("httpd","2.2.3-43.sl5.3",PKG_ARCH_DEFAULT); +"/software/packages"=pkg_repl("php"); + +# Example function +function show_things_view_for_stuff = { + thing = ARGV[0]; + foreach( i; mything; STUFF ) { + if ( thing == mything ) { + return( true ); + } else { + return SELF; + }; + }; + false; +}; + +variable HERE = <<EOF; +; This example demonstrates an in-line heredoc style config file +[main] +awesome = true +EOF + +variable small = false;#This should be highlighted normally again. diff --git a/tests/examplefiles/test.r3 b/tests/examplefiles/test.r3 index cad12a8d..707102db 100644 --- a/tests/examplefiles/test.r3 +++ b/tests/examplefiles/test.r3 @@ -1,3 +1,9 @@ +preface.... everything what is before header is not evaluated +so this should not be colorized: +1 + 2 + +REBOL [] ;<- this is minimal header, everything behind it must be colorized + ;## String tests ## print "Hello ^"World" ;<- with escaped char multiline-string: { @@ -52,15 +58,29 @@ type? #ff0000 ;== issue! to integer! (1 + (x / 4.5) * 1E-4) ;## some spec comments -comment now -comment 10 +1 + 1 +comment "aa" +2 + 2 +comment {aa} +3 + 3 +comment {a^{} +4 + 4 +comment {{}} +5 + 5 comment { - bla - bla + foo: 6 } -comment [ - quit -] +6 + 6 +comment [foo: 6] +7 + 7 +comment [foo: "[" ] +8 + 8 +comment [foo: {^{} ] +9 + 9 +comment [foo: {boo} ] +10 + 10 +comment 5-May-2014/11:17:34+2:00 +5-May-2014/11:17:34+2:00 11 + 11 ;## other tests ## ---: 1 diff --git a/tests/examplefiles/test.rsl b/tests/examplefiles/test.rsl new file mode 100644 index 00000000..d6c9fc9a --- /dev/null +++ b/tests/examplefiles/test.rsl @@ -0,0 +1,111 @@ +scheme COMPILER = +class + type + Prog == mk_Prog(stmt : Stmt), + + Stmt == + mk_Asgn(ide : Identifier, expr : Expr) | + mk_If(cond : Expr, s1 : Stmt, s2 : Stmt) | + mk_Seq(head : Stmt, last : Stmt), + + Expr == + mk_Const(const : Int) | + mk_Plus(fst : Expr, snd : Expr) | + mk_Id(ide : Identifier), + Identifier = Text + +type /* storage for program variables */ + `Sigma = Identifier -m-> Int + +value + m : Prog -> `Sigma -> `Sigma + m(p)(`sigma) is m(stmt(p))(`sigma), + + m : Stmt -> `Sigma -> `Sigma + m(s)(`sigma) is + case s of + mk_Asgn(i, e) -> `sigma !! [i +> m(e)(`sigma)], + mk_Seq(s1, s2) -> m(s2)(m(s1)(`sigma)), + mk_If(c, s1, s2) -> + if m(c)(`sigma) ~= 0 then m(s1)(`sigma) else m(s2)(`sigma) end + end, + + m : Expr -> `Sigma -> Int + m(e)(`sigma) is + case e of + mk_Const(n) -> n, + mk_Plus(e1, e2) -> m(e1)(`sigma) + m(e2)(`sigma), + mk_Id(id) -> if id isin dom `sigma then `sigma(id) else 0 end + end + +type + MProg = Inst-list, + Inst == + mk_Push(ide1 : Identifier) | + mk_Pop(Unit) | + mk_Add(Unit) | + mk_Cnst(val : Int) | + mk_Store(ide2 : Identifier) | + mk_Jumpfalse(off1 : Int) | + mk_Jump(off2 : Int) + + +/* An interpreter for SMALL instructions */ + +type Stack = Int-list +value + I : MProg >< Int >< Stack -> (`Sigma ->`Sigma) + I(mp, pc, s)(`sigma) is + if pc <= 0 \/ pc > len mp then `sigma else + case mp(pc) of + mk_Push(x) -> if x isin dom `sigma + then I(mp, pc + 1, <.`sigma(x).> ^ s)(`sigma) + else I(mp, pc + 1, <.0.> ^ s)(`sigma) end, + mk_Pop(()) -> if len s = 0 then `sigma + else I(mp, pc + 1, tl s)(`sigma) end, + mk_Cnst(n) -> I(mp, pc + 1, <.n.> ^ s)(`sigma), + mk_Add(()) -> if len s < 2 then `sigma + else I(mp, pc + 1,<.s(1) + s(2).> ^ tl tl s)(`sigma) end, + mk_Store(x) -> if len s = 0 then `sigma + else I(mp, pc + 1, s)(`sigma !! [x +> s(1)]) end, + mk_Jumpfalse(n) -> if len s = 0 then `sigma + elsif hd s ~= 0 then I(mp, pc + 1, s)(`sigma) + else I(mp, pc + n, s)(`sigma) end, + mk_Jump(n) -> I(mp, pc + n, s)(`sigma) + end + end + +value + comp_Prog : Prog -> MProg + comp_Prog(p) is comp_Stmt(stmt(p)), + + comp_Stmt : Stmt -> MProg + comp_Stmt(s) is + case s of + mk_Asgn(id, e) -> comp_Expr(e) ^ <. mk_Store(id), mk_Pop() .>, + mk_Seq(s1, s2) -> comp_Stmt(s1) ^ comp_Stmt(s2), + mk_If(e, s1, s2) -> + let + ce = comp_Expr(e), + cs1 = comp_Stmt(s1), cs2 = comp_Stmt(s2) + in + ce ^ + <. mk_Jumpfalse(len cs1 + 3) .> ^ + <. mk_Pop() .> ^ + cs1 ^ + <. mk_Jump(len cs2 + 2) .> ^ + <. mk_Pop() .> ^ + cs2 + end + end, + + comp_Expr : Expr -> MProg + comp_Expr(e) is + case e of + mk_Const(n) -> <. mk_Cnst(n) .>, + mk_Plus(e1, e2) -> + comp_Expr(e1) ^ comp_Expr(e2) ^ <. mk_Add() .>, + mk_Id(id) -> <. mk_Push(id) .> + end + +end diff --git a/tests/examplefiles/vimrc b/tests/examplefiles/vimrc new file mode 100644 index 00000000..d2f9cd1b --- /dev/null +++ b/tests/examplefiles/vimrc @@ -0,0 +1,21 @@ +" A comment + +:py print "py" +::pyt print 'pyt' + pyth print '''pyth''' + : pytho print "pytho" +python print """python""" + + : : python<<E OF +print """my script""" + +def MyFunc(str): + """ My Function """ + print str +E OF + +let py = 42 +echo py + +let foo = 42 +echo foo diff --git a/tests/string_asserts.py b/tests/string_asserts.py new file mode 100644 index 00000000..025a5281 --- /dev/null +++ b/tests/string_asserts.py @@ -0,0 +1,22 @@ +# -*- coding: utf-8 -*- +""" + Pygments string assert utility + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +class StringTests(object): + + def assertStartsWith(self, haystack, needle, msg=None): + if msg is None: + msg = "'{}' does not start with '{}'".format(haystack, needle) + if not haystack.startswith(needle): + raise(AssertionError(msg)) + + def assertEndsWith(self, haystack, needle, msg=None): + if msg is None: + msg = "'{}' does not end with '{}'".format(haystack, needle) + if not haystack.endswith(needle): + raise(AssertionError(msg)) diff --git a/tests/test_basic_api.py b/tests/test_basic_api.py index be7a4747..893fa90c 100644 --- a/tests/test_basic_api.py +++ b/tests/test_basic_api.py @@ -101,7 +101,7 @@ def test_lexer_options(): 'BashSessionLexer', 'LiterateHaskellLexer', 'LiterateAgdaLexer', 'PostgresConsoleLexer', 'ElixirConsoleLexer', 'JuliaConsoleLexer', 'RobotFrameworkLexer', 'DylanConsoleLexer', 'ShellSessionLexer', - 'LiterateIdrisLexer'): + 'LiterateIdrisLexer', 'LiterateCryptolLexer'): inst = cls(ensurenl=False) ensure(inst.get_tokens('a\nb'), 'a\nb') inst = cls(ensurenl=False, stripall=True) diff --git a/tests/test_cfm.py b/tests/test_cfm.py new file mode 100644 index 00000000..2ff25bd6 --- /dev/null +++ b/tests/test_cfm.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" + Basic ColdfusionHtmlLexer Test + ~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +import os + +from pygments.token import Token +from pygments.lexers import ColdfusionHtmlLexer + + +class ColdfusionHtmlLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = ColdfusionHtmlLexer() + + def testBasicComment(self): + fragment = u'<!--- cfcomment --->' + expected = [ + (Token.Text, u''), + (Token.Comment.Multiline, u'<!---'), + (Token.Comment.Multiline, u' cfcomment '), + (Token.Comment.Multiline, u'--->'), + (Token.Text, u'\n'), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + + def testNestedComment(self): + fragment = u'<!--- nested <!--- cfcomment ---> --->' + expected = [ + (Token.Text, u''), + (Token.Comment.Multiline, u'<!---'), + (Token.Comment.Multiline, u' nested '), + (Token.Comment.Multiline, u'<!---'), + (Token.Comment.Multiline, u' cfcomment '), + (Token.Comment.Multiline, u'--->'), + (Token.Comment.Multiline, u' '), + (Token.Comment.Multiline, u'--->'), + (Token.Text, u'\n'), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_clexer.py b/tests/test_clexer.py index c995bb2b..188569dc 100644 --- a/tests/test_clexer.py +++ b/tests/test_clexer.py @@ -9,8 +9,9 @@ import unittest import os +import textwrap -from pygments.token import Text, Number +from pygments.token import Text, Number, Token from pygments.lexers import CLexer @@ -29,3 +30,227 @@ class CLexerTest(unittest.TestCase): wanted.append((Text, ' ')) wanted = [(Text, '')] + wanted[:-1] + [(Text, '\n')] self.assertEqual(list(self.lexer.get_tokens(code)), wanted) + + def testSwitch(self): + fragment = u'''\ + int main() + { + switch (0) + { + case 0: + default: + ; + } + } + ''' + tokens = [ + (Token.Text, u''), + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Text, u''), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u''), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'switch'), + (Token.Text, u' '), + (Token.Punctuation, u'('), + (Token.Literal.Number.Integer, u'0'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'case'), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'0'), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'default'), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testSwitchSpaceBeforeColon(self): + fragment = u'''\ + int main() + { + switch (0) + { + case 0 : + default : + ; + } + } + ''' + tokens = [ + (Token.Text, u''), + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Text, u''), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u''), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'switch'), + (Token.Text, u' '), + (Token.Punctuation, u'('), + (Token.Literal.Number.Integer, u'0'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'case'), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'0'), + (Token.Text, u' '), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'default'), + (Token.Text, u' '), + (Token.Operator, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testLabel(self): + fragment = u'''\ + int main() + { + foo: + goto foo; + } + ''' + tokens = [ + (Token.Text, u''), + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Text, u''), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u''), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Name.Label, u'foo'), + (Token.Punctuation, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'goto'), + (Token.Text, u' '), + (Token.Name, u'foo'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testLabelSpaceBeforeColon(self): + fragment = u'''\ + int main() + { + foo : + goto foo; + } + ''' + tokens = [ + (Token.Text, u''), + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Text, u''), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u''), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Name.Label, u'foo'), + (Token.Text, u' '), + (Token.Punctuation, u':'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'goto'), + (Token.Text, u' '), + (Token.Name, u'foo'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) + + def testLabelFollowedByStatement(self): + fragment = u'''\ + int main() + { + foo:return 0; + goto foo; + } + ''' + tokens = [ + (Token.Text, u''), + (Token.Keyword.Type, u'int'), + (Token.Text, u' '), + (Token.Name.Function, u'main'), + (Token.Text, u''), + (Token.Punctuation, u'('), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + (Token.Text, u''), + (Token.Punctuation, u'{'), + (Token.Text, u'\n'), + (Token.Name.Label, u'foo'), + (Token.Punctuation, u':'), + (Token.Keyword, u'return'), + (Token.Text, u' '), + (Token.Literal.Number.Integer, u'0'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u' '), + (Token.Keyword, u'goto'), + (Token.Text, u' '), + (Token.Name, u'foo'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Punctuation, u'}'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(tokens, list(self.lexer.get_tokens(textwrap.dedent(fragment)))) diff --git a/tests/test_lexers_other.py b/tests/test_lexers_other.py index 1e420c77..91b0dc70 100644 --- a/tests/test_lexers_other.py +++ b/tests/test_lexers_other.py @@ -49,20 +49,20 @@ class RexxLexerTest(unittest.TestCase): self.assertAlmostEqual(1.0, RexxLexer.analyse_text('''/* Rexx */ say "hello world"''')) - self.assertLess(0.5, - RexxLexer.analyse_text('/* */\n' + val = RexxLexer.analyse_text('/* */\n' 'hello:pRoceduRe\n' - ' say "hello world"')) - self.assertLess(0.2, - RexxLexer.analyse_text('''/* */ + ' say "hello world"') + self.assertTrue(val > 0.5, val) + val = RexxLexer.analyse_text('''/* */ if 1 > 0 then do say "ok" end else do say "huh?" - end''')) - self.assertLess(0.2, - RexxLexer.analyse_text('''/* */ + end''') + self.assertTrue(val > 0.2, val) + val = RexxLexer.analyse_text('''/* */ greeting = "hello world!" parse value greeting "hello" name "!" - say name''')) + say name''') + self.assertTrue(val > 0.2, val) diff --git a/tests/test_objectiveclexer.py b/tests/test_objectiveclexer.py new file mode 100644 index 00000000..46fdb6d2 --- /dev/null +++ b/tests/test_objectiveclexer.py @@ -0,0 +1,91 @@ +# -*- coding: utf-8 -*- +""" + Basic CLexer Test + ~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +import os + +from pygments.token import Token +from pygments.lexers import ObjectiveCLexer + + +class ObjectiveCLexerTest(unittest.TestCase): + + def setUp(self): + self.lexer = ObjectiveCLexer() + + def testLiteralNumberInt(self): + fragment = u'@(1);\n' + expected = [ + (Token.Text, u''), + (Token.Literal, u'@('), + (Token.Literal.Number.Integer, u'1'), + (Token.Literal, u')'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + + def testLiteralNumberExpression(self): + fragment = u'@(1+2);\n' + expected = [ + (Token.Text, u''), + (Token.Literal, u'@('), + (Token.Literal.Number.Integer, u'1'), + (Token.Operator, u'+'), + (Token.Literal.Number.Integer, u'2'), + (Token.Literal, u')'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + + def testLiteralNumberNestedExpression(self): + fragment = u'@(1+(2+3));\n' + expected = [ + (Token.Text, u''), + (Token.Literal, u'@('), + (Token.Literal.Number.Integer, u'1'), + (Token.Operator, u'+'), + (Token.Punctuation, u'('), + (Token.Literal.Number.Integer, u'2'), + (Token.Operator, u'+'), + (Token.Literal.Number.Integer, u'3'), + (Token.Punctuation, u')'), + (Token.Literal, u')'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + + def testLiteralNumberBool(self): + fragment = u'@NO;\n' + expected = [ + (Token.Text, u''), + (Token.Literal.Number, u'@NO'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + + def testLieralNumberBoolExpression(self): + fragment = u'@(YES);\n' + expected = [ + (Token.Text, u''), + (Token.Literal, u'@('), + (Token.Name.Builtin, u'YES'), + (Token.Literal, u')'), + (Token.Punctuation, u';'), + (Token.Text, u'\n'), + (Token.Text, u''), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) diff --git a/tests/test_qbasiclexer.py b/tests/test_qbasiclexer.py new file mode 100644 index 00000000..1b81b643 --- /dev/null +++ b/tests/test_qbasiclexer.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +""" + Tests for QBasic + ~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import glob +import os +import unittest + +from pygments.token import Token +from pygments.lexers.qbasic import QBasicLexer + +class QBasicTest(unittest.TestCase): + def setUp(self): + self.lexer = QBasicLexer() + self.maxDiff = None + + def testKeywordsWithDollar(self): + fragment = u'DIM x\nx = RIGHT$("abc", 1)\n' + expected = [ + (Token.Keyword.Declaration, u'DIM'), + (Token.Text.Whitespace, u' '), + (Token.Name.Variable.Global, u'x'), + (Token.Text, u'\n'), + (Token.Name.Variable.Global, u'x'), + (Token.Text.Whitespace, u' '), + (Token.Operator, u'='), + (Token.Text.Whitespace, u' '), + (Token.Keyword.Reserved, u'RIGHT$'), + (Token.Punctuation, u'('), + (Token.Literal.String.Double, u'"abc"'), + (Token.Punctuation, u','), + (Token.Text.Whitespace, u' '), + (Token.Literal.Number.Integer.Long, u'1'), + (Token.Punctuation, u')'), + (Token.Text, u'\n'), + ] + self.assertEqual(expected, list(self.lexer.get_tokens(fragment))) + diff --git a/tests/test_rtf_formatter.py b/tests/test_rtf_formatter.py new file mode 100644 index 00000000..30b136fd --- /dev/null +++ b/tests/test_rtf_formatter.py @@ -0,0 +1,109 @@ +# -*- coding: utf-8 -*- +""" + Pygments RTF formatter tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +from string_asserts import StringTests + +from pygments.util import StringIO +from pygments.formatters import RtfFormatter +from pygments.lexers.special import TextLexer + +class RtfFormatterTest(StringTests, unittest.TestCase): + foot = (r'\par' '\n' r'}') + + def _escape(self, string): + return(string.replace("\n", r"\n")) + + def _build_message(self, *args, **kwargs): + string = kwargs.get('string', None) + t = self._escape(kwargs.get('t', '')) + expected = self._escape(kwargs.get('expected', '')) + result = self._escape(kwargs.get('result', '')) + + if string is None: + string = (u"The expected output of '{t}'\n" + u"\t\tShould be '{expected}'\n" + u"\t\tActually outputs '{result}'\n" + u"\t(WARNING: Partial Output of Result!)") + + end = -(len(self._escape(self.foot))) + start = end-len(expected) + + return string.format(t=t, + result = result[start:end], + expected = expected) + + def format_rtf(self, t): + tokensource = list(TextLexer().get_tokens(t)) + fmt = RtfFormatter() + buf = StringIO() + fmt.format(tokensource, buf) + result = buf.getvalue() + buf.close() + return result + + def test_rtf_header(self): + t = u'' + result = self.format_rtf(t) + expected = r'{\rtf1\ansi\uc0' + msg = (u"RTF documents are expected to start with '{expected}'\n" + u"\t\tStarts intead with '{result}'\n" + u"\t(WARNING: Partial Output of Result!)".format( + expected = expected, + result = result[:len(expected)])) + self.assertStartsWith(result, expected, msg) + + def test_rtf_footer(self): + t = u'' + result = self.format_rtf(t) + expected = self.foot + msg = (u"RTF documents are expected to end with '{expected}'\n" + u"\t\tEnds intead with '{result}'\n" + u"\t(WARNING: Partial Output of Result!)".format( + expected = self._escape(expected), + result = self._escape(result[-len(expected):]))) + self.assertEndsWith(result, expected, msg) + + def test_ascii_characters(self): + t = u'a b c d ~' + result = self.format_rtf(t) + expected = (r'a b c d ~') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) + + def test_escape_characters(self): + t = u'\ {{' + result = self.format_rtf(t) + expected = (r'\\ \{\{') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) + + def test_single_characters(self): + t = u'â € ¤ каждой' + result = self.format_rtf(t) + expected = (r'{\u226} {\u8364} {\u164} ' + r'{\u1082}{\u1072}{\u1078}{\u1076}{\u1086}{\u1081}') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) + + def test_double_characters(self): + t = u'က 힣 ↕ ↕︎ 鼖' + result = self.format_rtf(t) + expected = (r'{\u4096} {\u55203} {\u8597} ' + r'{\u8597}{\u65038} {\u55422}{\u56859}') + if not result.endswith(self.foot): + return(unittest.skip('RTF Footer incorrect')) + msg = self._build_message(t=t, result=result, expected=expected) + self.assertEndsWith(result, expected+self.foot, msg) diff --git a/tests/test_string_asserts.py b/tests/test_string_asserts.py new file mode 100644 index 00000000..0beed15c --- /dev/null +++ b/tests/test_string_asserts.py @@ -0,0 +1,39 @@ +# -*- coding: utf-8 -*- +""" + Pygments string assert utility tests + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :copyright: Copyright 2006-2014 by the Pygments team, see AUTHORS. + :license: BSD, see LICENSE for details. +""" + +import unittest +from string_asserts import StringTests + +class TestStringTests(StringTests, unittest.TestCase): + + def test_startswith_correct(self): + self.assertStartsWith("AAA", "A") + + # @unittest.expectedFailure not supported by nose + def test_startswith_incorrect(self): + with self.assertRaises(AssertionError): + self.assertStartsWith("AAA", "B") + + # @unittest.expectedFailure not supported by nose + def test_startswith_short(self): + with self.assertRaises(AssertionError): + self.assertStartsWith("A", "AA") + + def test_endswith_correct(self): + self.assertEndsWith("AAA", "A") + + # @unittest.expectedFailure not supported by nose + def test_endswith_incorrect(self): + with self.assertRaises(AssertionError): + self.assertEndsWith("AAA", "B") + + # @unittest.expectedFailure not supported by nose + def test_endswith_short(self): + with self.assertRaises(AssertionError): + self.assertEndsWith("A", "AA") |