From ae448d124630513e63151850e8a458b68ece33b6 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Wed, 5 Oct 2016 17:14:50 -0700 Subject: Add HLSL lexer --- pygments/lexers/_mapping.py | 1 + pygments/lexers/graphics.py | 147 +++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 147 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index a6097b1c..6412d567 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -174,6 +174,7 @@ LEXERS = { 'GosuTemplateLexer': ('pygments.lexers.jvm', 'Gosu Template', ('gst',), ('*.gst',), ('text/x-gosu-template',)), 'GroffLexer': ('pygments.lexers.markup', 'Groff', ('groff', 'nroff', 'man'), ('*.[1234567]', '*.man'), ('application/x-troff', 'text/troff')), 'GroovyLexer': ('pygments.lexers.jvm', 'Groovy', ('groovy',), ('*.groovy', '*.gradle'), ('text/x-groovy',)), + 'HLSLShaderLexer': ('pygments.lexers.graphics', 'HLSL', ('hlsl',), ('*.hlsl', '*.hlsli'), ('text/x-hlsl',)), 'HamlLexer': ('pygments.lexers.html', 'Haml', ('haml',), ('*.haml',), ('text/x-haml',)), 'HandlebarsHtmlLexer': ('pygments.lexers.templates', 'HTML+Handlebars', ('html+handlebars',), ('*.handlebars', '*.hbs'), ('text/html+handlebars', 'text/x-handlebars-template')), 'HandlebarsLexer': ('pygments.lexers.templates', 'Handlebars', ('handlebars',), (), ()), diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index b40e0286..a5da2375 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -15,7 +15,7 @@ from pygments.token import Text, Comment, Operator, Keyword, Name, \ Number, Punctuation, String __all__ = ['GLShaderLexer', 'PostScriptLexer', 'AsymptoteLexer', 'GnuplotLexer', - 'PovrayLexer'] + 'PovrayLexer', 'HLSLShaderLexer'] class GLShaderLexer(RegexLexer): @@ -75,6 +75,151 @@ class GLShaderLexer(RegexLexer): } +class HLSLShaderLexer(RegexLexer): + """ + HLSL (Microsoft Direct3D Shader) lexer. + + .. versionadded:: 2.2 + """ + name = 'HLSL' + aliases = ['hlsl'] + filenames = ['*.hlsl', '*.hlsli'] + mimetypes = ['text/x-hlsl'] + + tokens = { + 'root': [ + (r'^#.*', Comment.Preproc), + (r'//.*', Comment.Single), + (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), + (r'\+|-|~|!=?|\*|/|%|<<|>>|<=?|>=?|==?|&&?|\^|\|\|?', + Operator), + (r'[?:]', Operator), # quick hack for ternary + (r'\bdefined\b', Operator), + (r'[;{}(),.\[\]]', Punctuation), + # FIXME when e is present, no decimal point needed + (r'[+-]?\d*\.\d+([eE][-+]?\d+)?f?', Number.Float), + (r'[+-]?\d+\.\d*([eE][-+]?\d+)?f?', Number.Float), + (r'0[xX][0-9a-fA-F]*', Number.Hex), + (r'0[0-7]*', Number.Oct), + (r'[1-9][0-9]*', Number.Integer), + (words(( + 'asm','asm_fragment','break','case','cbuffer','centroid','class', + 'column_major','compile','compile_fragment','const','continue', + 'default','discard','do','else','export','extern','for','fxgroup', + 'globallycoherent','groupshared','if','in','inline','inout', + 'interface','line','lineadj','linear','namespace','nointerpolation', + 'noperspective','NULL','out','packoffset','pass','pixelfragment', + 'point','precise','return','register','row_major','sample', + 'sampler','shared','stateblock','stateblock_state','static', + 'struct','switch','tbuffer','technique','technique10', + 'technique11','texture','typedef','triangle','triangleadj', + 'uniform','vertexfragment','volatile','while'), + prefix=r'\b', suffix=r'\b'), + Keyword), + (words(('true','false'), prefix=r'\b', suffix=r'\b'), + Keyword.Constant), + (words(( + 'auto','catch','char','const_cast','delete','dynamic_cast','enum', + 'explicit','friend','goto','long','mutable','new','operator', + 'private','protected','public','reinterpret_cast','short','signed', + 'sizeof','static_cast','template','this','throw','try','typename', + 'union','unsigned','using','virtual'), + prefix=r'\b', suffix=r'\b'), + Keyword.Reserved), + (words(( + 'dword','matrix','snorm','string','unorm','unsigned','void','vector', + 'BlendState','Buffer','ByteAddressBuffer','ComputeShader', + 'DepthStencilState','DepthStencilView','DomainShader', + 'GeometryShader','HullShader','InputPatch','LineStream', + 'OutputPatch','PixelShader','PointStream','RasterizerState', + 'RenderTargetView','RasterizerOrderedBuffer', + 'RasterizerOrderedByteAddressBuffer', + 'RasterizerOrderedStructuredBuffer','RasterizerOrderedTexture1D', + 'RasterizerOrderedTexture1DArray','RasterizerOrderedTexture2D', + 'RasterizerOrderedTexture2DArray','RasterizerOrderedTexture3D', + 'RWBuffer','RWByteAddressBuffer','RWStructuredBuffer', + 'RWTexture1D','RWTexture1DArray','RWTexture2D','RWTexture2DArray', + 'RWTexture3D','SamplerState','SamplerComparisonState', + 'StructuredBuffer','Texture1D','Texture1DArray','Texture2D', + 'Texture2DArray','Texture2DMS','Texture2DMSArray','Texture3D', + 'TextureCube','TextureCubeArray','TriangleStream','VertexShader'), + prefix=r'\b', suffix=r'\b'), + Keyword.Type), + (words(( + 'bool','double','float','int','half','min16float','min10float', + 'min16int','min12int','min16uint','uint'), + prefix=r'\b', suffix=r'([1-4](x[1-4])?)?\b'), + Keyword.Type), # vector and matrix types + (words(( + 'abort','abs','acos','all','AllMemoryBarrier', + 'AllMemoryBarrierWithGroupSync','any','AppendStructuredBuffer', + 'asdouble','asfloat','asin','asint','asuint','asuint','atan', + 'atan2','ceil','CheckAccessFullyMapped','clamp','clip', + 'CompileShader','ConsumeStructuredBuffer','cos','cosh','countbits', + 'cross','D3DCOLORtoUBYTE4','ddx','ddx_coarse','ddx_fine','ddy', + 'ddy_coarse','ddy_fine','degrees','determinant', + 'DeviceMemoryBarrier','DeviceMemoryBarrierWithGroupSync','distance', + 'dot','dst','errorf','EvaluateAttributeAtCentroid', + 'EvaluateAttributeAtSample','EvaluateAttributeSnapped','exp', + 'exp2','f16tof32','f32tof16','faceforward','firstbithigh', + 'firstbitlow','floor','fma','fmod','frac','frexp','fwidth', + 'GetRenderTargetSampleCount','GetRenderTargetSamplePosition', + 'GlobalOrderedCountIncrement','GroupMemoryBarrier', + 'GroupMemoryBarrierWithGroupSync','InterlockedAdd','InterlockedAnd', + 'InterlockedCompareExchange','InterlockedCompareStore', + 'InterlockedExchange','InterlockedMax','InterlockedMin', + 'InterlockedOr','InterlockedXor','isfinite','isinf','isnan', + 'ldexp','length','lerp','lit','log','log10','log2','mad','max', + 'min','modf','msad4','mul','noise','normalize','pow','printf', + 'Process2DQuadTessFactorsAvg','Process2DQuadTessFactorsMax', + 'Process2DQuadTessFactorsMin','ProcessIsolineTessFactors', + 'ProcessQuadTessFactorsAvg','ProcessQuadTessFactorsMax', + 'ProcessQuadTessFactorsMin','ProcessTriTessFactorsAvg', + 'ProcessTriTessFactorsMax','ProcessTriTessFactorsMin', + 'QuadReadLaneAt','QuadSwapX','QuadSwapY','radians','rcp', + 'reflect','refract','reversebits','round','rsqrt','saturate', + 'sign','sin','sincos','sinh','smoothstep','sqrt','step','tan', + 'tanh','tex1D','tex1D','tex1Dbias','tex1Dgrad','tex1Dlod', + 'tex1Dproj','tex2D','tex2D','tex2Dbias','tex2Dgrad','tex2Dlod', + 'tex2Dproj','tex3D','tex3D','tex3Dbias','tex3Dgrad','tex3Dlod', + 'tex3Dproj','texCUBE','texCUBE','texCUBEbias','texCUBEgrad', + 'texCUBElod','texCUBEproj','transpose','trunc','WaveAllBitAnd', + 'WaveAllMax','WaveAllMin','WaveAllBitOr','WaveAllBitXor', + 'WaveAllEqual','WaveAllProduct','WaveAllSum','WaveAllTrue', + 'WaveAnyTrue','WaveBallot','WaveGetLaneCount','WaveGetLaneIndex', + 'WaveGetOrderedIndex','WaveIsHelperLane','WaveOnce', + 'WavePrefixProduct','WavePrefixSum','WaveReadFirstLane', + 'WaveReadLaneAt'), + prefix=r'\b', suffix=r'\b'), + Name.Builtin), # built-in functions + (words(( + 'SV_ClipDistance','SV_ClipDistance0','SV_ClipDistance1', + 'SV_Culldistance','SV_CullDistance0','SV_CullDistance1', + 'SV_Coverage','SV_Depth','SV_DepthGreaterEqual', + 'SV_DepthLessEqual','SV_DispatchThreadID','SV_DomainLocation', + 'SV_GroupID','SV_GroupIndex','SV_GroupThreadID','SV_GSInstanceID', + 'SV_InnerCoverage','SV_InsideTessFactor','SV_InstanceID', + 'SV_IsFrontFace','SV_OutputControlPointID','SV_Position', + 'SV_PrimitiveID','SV_RenderTargetArrayIndex','SV_SampleIndex', + 'SV_StencilRef','SV_TessFactor','SV_VertexID', + 'SV_ViewportArrayIndex'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # system-value semantics + (r'\bSV_Target[0-7]?\b', Name.Decorator), + (words(( + 'allow_uav_condition','branch','call','domain','earlydepthstencil', + 'fastopt','flatten','forcecase','instance','loop','maxtessfactor', + 'numthreads','outputcontrolpoints','outputtopology','partitioning', + 'patchconstantfunc','unroll'), + prefix=r'\b', suffix=r'\b'), + Name.Decorator), # attributes + (r'[a-zA-Z_]\w*', Name), + (r'\\$', Comment.Preproc), # backslash at end of line -- usually macro continuation + (r'\s+', Text), + ], + } + + class PostScriptLexer(RegexLexer): """ Lexer for PostScript files. -- cgit v1.2.1 From c22e58c9e4c2fd191782fc9659c0044a9c13ead3 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Wed, 5 Oct 2016 17:15:01 -0700 Subject: Add myself to AUTHORS --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index 66377ead..18051bb4 100644 --- a/AUTHORS +++ b/AUTHORS @@ -159,6 +159,7 @@ Other contributors, listed alphabetically, are: * Elias Rabel -- Fortran fixed form lexer * raichoo -- Idris lexer * Kashif Rasul -- CUDA lexer +* Nathan Reed -- HLSL lexer * Justin Reidy -- MXML lexer * Norman Richards -- JSON lexer * Corey Richardson -- Rust lexer updates -- cgit v1.2.1 From c8f8ac0ef923681e47ff852809c28978fa790176 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Wed, 5 Oct 2016 17:36:45 -0700 Subject: Add example file for HLSL --- tests/examplefiles/example.hlsl | 157 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 157 insertions(+) create mode 100644 tests/examplefiles/example.hlsl diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl new file mode 100644 index 00000000..77f1fa49 --- /dev/null +++ b/tests/examplefiles/example.hlsl @@ -0,0 +1,157 @@ +// A few random snippets of HLSL shader code I gathered... + +[numthreads(256, 1, 1)] +void cs_main(uint3 threadId : SV_DispatchThreadID) +{ + // Seed the PRNG using the thread ID + rng_state = threadId.x; + + // Generate a few numbers... + uint r0 = rand_xorshift(); + uint r1 = rand_xorshift(); + // Do some stuff with them... + + // Generate a random float in [0, 1)... + float f0 = float(rand_xorshift()) * (1.0 / 4294967296.0); + + // ...etc. +} + +// Constant buffer of parameters +cbuffer IntegratorParams : register(b0) +{ + float2 specPow; // Spec powers in XY directions (equal for isotropic BRDFs) + float3 L; // Unit vector toward light + int2 cThread; // Total threads launched in XY dimensions + int2 xyOutput; // Where in the output buffer to store the result +} + +static const float pi = 3.141592654; + +float AshikhminShirleyNDF(float3 H) +{ + float normFactor = sqrt((specPow.x + 2.0f) * (specPow.y + 2.0)) * (0.5f / pi); + float NdotH = H.z; + float2 Hxy = normalize(H.xy); + return normFactor * pow(NdotH, dot(specPow, Hxy * Hxy)); +} + +float BeckmannNDF(float3 H) +{ + float glossFactor = specPow.x * 0.5f + 1.0f; // This is 1/m^2 in the usual Beckmann formula + float normFactor = glossFactor * (1.0f / pi); + float NdotHSq = H.z * H.z; + return normFactor / (NdotHSq * NdotHSq) * exp(glossFactor * (1.0f - 1.0f / NdotHSq)); +} + +// Output buffer for compute shader (actually float, but must be declared as uint +// for atomic operations to work) +globallycoherent RWTexture2D o_data : register(u0); + +// Sum up the outputs of all threads and store to the output location +static const uint threadGroupSize2D = 16; +static const uint threadGroupSize1D = threadGroupSize2D * threadGroupSize2D; +groupshared float g_partialSums[threadGroupSize1D]; +void SumAcrossThreadsAndStore(float value, uint iThreadInGroup) +{ + // First reduce within the threadgroup: partial sums of 2, 4, 8... elements + // are calculated by 1/2, 1/4, 1/8... of the threads, always keeping the + // active threads at the front of the group to minimize divergence. + + // NOTE: there are faster ways of doing this...but this is simple to code + // and good enough. + + g_partialSums[iThreadInGroup] = value; + GroupMemoryBarrierWithGroupSync(); + + [unroll] for (uint i = threadGroupSize1D / 2; i > 0; i /= 2) + { + if (iThreadInGroup < i) + { + g_partialSums[iThreadInGroup] += g_partialSums[iThreadInGroup + i]; + } + GroupMemoryBarrierWithGroupSync(); + } + + // Then reduce across threadgroups: one thread from each group adds the group + // total to the final output location, using a software transactional memory + // style since D3D11 doesn't support atomic add on floats. + // (Assumes the output value has been cleared to zero beforehand.) + + if (iThreadInGroup == 0) + { + float threadGroupSum = g_partialSums[0]; + uint outputValueRead = o_data[xyOutput]; + while (true) + { + uint newOutputValue = asuint(asfloat(outputValueRead) + threadGroupSum); + uint previousOutputValue; + InterlockedCompareExchange( + o_data[xyOutput], outputValueRead, newOutputValue, previousOutputValue); + if (previousOutputValue == outputValueRead) + break; + outputValueRead = previousOutputValue; + } + } +} + +void main( + in Vertex i_vtx, + out Vertex o_vtx, + out float3 o_vecCamera : CAMERA, + out float4 o_uvzwShadow : UVZW_SHADOW, + out float4 o_posClip : SV_Position) +{ + o_vtx = i_vtx; + o_vecCamera = g_posCamera - i_vtx.m_pos; + o_uvzwShadow = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToUvzwShadow); + o_posClip = mul(float4(i_vtx.m_pos, 1.0), g_matWorldToClip); +} + +#pragma pack_matrix(row_major) + +struct Vertex +{ + float3 m_pos : POSITION; + float3 m_normal : NORMAL; + float2 m_uv : UV; +}; + +cbuffer CBFrame : CB_FRAME // matches struct CBFrame in test.cpp +{ + float4x4 g_matWorldToClip; + float4x4 g_matWorldToUvzwShadow; + float3x3 g_matWorldToUvzShadowNormal; + float3 g_posCamera; + + float3 g_vecDirectionalLight; + float3 g_rgbDirectionalLight; + + float2 g_dimsShadowMap; + float g_normalOffsetShadow; + float g_shadowSharpening; + + float g_exposure; // Exposure multiplier +} + +Texture2D g_texDiffuse : register(t0); +SamplerState g_ss : register(s0); + +void main( + in Vertex i_vtx, + in float3 i_vecCamera : CAMERA, + in float4 i_uvzwShadow : UVZW_SHADOW, + out float3 o_rgb : SV_Target) +{ + float3 normal = normalize(i_vtx.m_normal); + + // Sample shadow map + float shadow = EvaluateShadow(i_uvzwShadow, normal); + + // Evaluate diffuse lighting + float3 diffuseColor = g_texDiffuse.Sample(g_ss, i_vtx.m_uv); + float3 diffuseLight = g_rgbDirectionalLight * (shadow * saturate(dot(normal, g_vecDirectionalLight))); + diffuseLight += SimpleAmbient(normal); + + o_rgb = diffuseColor * diffuseLight; +} -- cgit v1.2.1 From 7fd626cad80ee0257ac4ac5b1365d5cbb52ac247 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Thu, 29 Dec 2016 17:48:53 -0800 Subject: Add string literal parsing to HLSL lexer (copied from the one for C++). - Also added a snippet to the example file where a string shows up (uncommon in HLSL). --- pygments/lexers/graphics.py | 9 +++++++++ tests/examplefiles/example.hlsl | 11 +++++++++++ 2 files changed, 20 insertions(+) diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index a5da2375..b47e12b1 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -102,6 +102,7 @@ class HLSLShaderLexer(RegexLexer): (r'0[xX][0-9a-fA-F]*', Number.Hex), (r'0[0-7]*', Number.Oct), (r'[1-9][0-9]*', Number.Integer), + (r'"', String, 'string'), (words(( 'asm','asm_fragment','break','case','cbuffer','centroid','class', 'column_major','compile','compile_fragment','const','continue', @@ -217,6 +218,14 @@ class HLSLShaderLexer(RegexLexer): (r'\\$', Comment.Preproc), # backslash at end of line -- usually macro continuation (r'\s+', Text), ], + 'string': [ + (r'"', String, '#pop'), + (r'\\([\\abfnrtv"\']|x[a-fA-F0-9]{2,4}|' + r'u[a-fA-F0-9]{4}|U[a-fA-F0-9]{8}|[0-7]{1,3})', String.Escape), + (r'[^\\"\n]+', String), # all other characters + (r'\\\n', String), # line continuation + (r'\\', String), # stray backslash + ], } diff --git a/tests/examplefiles/example.hlsl b/tests/examplefiles/example.hlsl index 77f1fa49..21d0a672 100644 --- a/tests/examplefiles/example.hlsl +++ b/tests/examplefiles/example.hlsl @@ -155,3 +155,14 @@ void main( o_rgb = diffuseColor * diffuseLight; } + +[domain("quad")] +void ds( + in float edgeFactors[4] : SV_TessFactor, + in float insideFactors[2] : SV_InsideTessFactor, + in OutputPatch inp, + in float2 uv : SV_DomainLocation, + out float4 o_pos : SV_Position) +{ + o_pos = lerp(lerp(inp[0].pos, inp[1].pos, uv.x), lerp(inp[2].pos, inp[3].pos, uv.x), uv.y); +} -- cgit v1.2.1 From 5521b1c682a4e5078c5c0d7bb3122a2e18f81f30 Mon Sep 17 00:00:00 2001 From: Nathan Reed Date: Thu, 29 Dec 2016 17:51:30 -0800 Subject: Fix tabs --- pygments/lexers/graphics.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/graphics.py b/pygments/lexers/graphics.py index b47e12b1..d62b64ba 100644 --- a/pygments/lexers/graphics.py +++ b/pygments/lexers/graphics.py @@ -102,7 +102,7 @@ class HLSLShaderLexer(RegexLexer): (r'0[xX][0-9a-fA-F]*', Number.Hex), (r'0[0-7]*', Number.Oct), (r'[1-9][0-9]*', Number.Integer), - (r'"', String, 'string'), + (r'"', String, 'string'), (words(( 'asm','asm_fragment','break','case','cbuffer','centroid','class', 'column_major','compile','compile_fragment','const','continue', -- cgit v1.2.1 From afcaa7d6ac6c676501446efa6f1a7de0d4eae86c Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Wed, 25 Jan 2017 07:58:50 +0100 Subject: PR#662: catch IOError from pkg_resources import --- pygments/plugin.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pygments/plugin.py b/pygments/plugin.py index 7987d646..08d9b5b4 100644 --- a/pygments/plugin.py +++ b/pygments/plugin.py @@ -40,14 +40,16 @@ FORMATTER_ENTRY_POINT = 'pygments.formatters' STYLE_ENTRY_POINT = 'pygments.styles' FILTER_ENTRY_POINT = 'pygments.filters' + def iter_entry_points(group_name): try: import pkg_resources - except ImportError: + except (ImportError, IOError): return [] return pkg_resources.iter_entry_points(group_name) + def find_plugin_lexers(): for entrypoint in iter_entry_points(LEXER_ENTRY_POINT): yield entrypoint.load() -- cgit v1.2.1 From 05f1ff72c4b5b665e1e39f6d09ac91aac27bd7ea Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Fri, 27 Jan 2017 07:22:40 +0100 Subject: Fixes #1320: restore Py2.6 compatibility --- pygments/lexers/_lua_builtins.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/_lua_builtins.py b/pygments/lexers/_lua_builtins.py index c60bf5a2..0561725d 100644 --- a/pygments/lexers/_lua_builtins.py +++ b/pygments/lexers/_lua_builtins.py @@ -288,7 +288,7 @@ if __name__ == '__main__': # pragma: no cover print('>> %s' % full_function_name) m = get_function_module(full_function_name) modules.setdefault(m, []).append(full_function_name) - modules = {k: tuple(v) for k, v in modules.iteritems()} + modules = dict((k, tuple(v)) for k, v in modules.iteritems()) regenerate(__file__, modules) -- cgit v1.2.1 From 43fcdda784228cf8c60287c4c64a575738653d94 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Fri, 27 Jan 2017 15:12:54 +0100 Subject: Add rs alias for Rust. --- pygments/lexers/_mapping.py | 4 ++-- pygments/lexers/rust.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index ea54241c..3b747036 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -367,7 +367,7 @@ LEXERS = { 'RtsLexer': ('pygments.lexers.trafficscript', 'TrafficScript', ('rts', 'trafficscript'), ('*.rts',), ()), 'RubyConsoleLexer': ('pygments.lexers.ruby', 'Ruby irb session', ('rbcon', 'irb'), (), ('text/x-ruby-shellsession',)), 'RubyLexer': ('pygments.lexers.ruby', 'Ruby', ('rb', 'ruby', 'duby'), ('*.rb', '*.rbw', 'Rakefile', '*.rake', '*.gemspec', '*.rbx', '*.duby', 'Gemfile'), ('text/x-ruby', 'application/x-ruby')), - 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust',), ('*.rs', '*.rs.in'), ('text/rust',)), + 'RustLexer': ('pygments.lexers.rust', 'Rust', ('rust', 'rs'), ('*.rs', '*.rs.in'), ('text/rust',)), 'SASLexer': ('pygments.lexers.sas', 'SAS', ('sas',), ('*.SAS', '*.sas'), ('text/x-sas', 'text/sas', 'application/x-sas')), 'SLexer': ('pygments.lexers.r', 'S', ('splus', 's', 'r'), ('*.S', '*.R', '.Rhistory', '.Rprofile', '.Renviron'), ('text/S-plus', 'text/S', 'text/x-r-source', 'text/x-r', 'text/x-R', 'text/x-r-history', 'text/x-r-profile')), 'SMLLexer': ('pygments.lexers.ml', 'Standard ML', ('sml',), ('*.sml', '*.sig', '*.fun'), ('text/x-standardml', 'application/x-standardml')), @@ -417,7 +417,7 @@ LEXERS = { 'TurtleLexer': ('pygments.lexers.rdf', 'Turtle', ('turtle',), ('*.ttl',), ('text/turtle', 'application/x-turtle')), 'TwigHtmlLexer': ('pygments.lexers.templates', 'HTML+Twig', ('html+twig',), ('*.twig',), ('text/html+twig',)), 'TwigLexer': ('pygments.lexers.templates', 'Twig', ('twig',), (), ('application/x-twig',)), - 'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts',), ('text/x-typescript',)), + 'TypeScriptLexer': ('pygments.lexers.javascript', 'TypeScript', ('ts', 'typescript'), ('*.ts', '*.tsx'), ('text/x-typescript',)), 'TypoScriptCssDataLexer': ('pygments.lexers.typoscript', 'TypoScriptCssData', ('typoscriptcssdata',), (), ()), 'TypoScriptHtmlDataLexer': ('pygments.lexers.typoscript', 'TypoScriptHtmlData', ('typoscripthtmldata',), (), ()), 'TypoScriptLexer': ('pygments.lexers.typoscript', 'TypoScript', ('typoscript',), ('*.ts', '*.txt'), ('text/x-typoscript',)), diff --git a/pygments/lexers/rust.py b/pygments/lexers/rust.py index 6914f54d..10097fba 100644 --- a/pygments/lexers/rust.py +++ b/pygments/lexers/rust.py @@ -24,7 +24,7 @@ class RustLexer(RegexLexer): """ name = 'Rust' filenames = ['*.rs', '*.rs.in'] - aliases = ['rust'] + aliases = ['rust', 'rs'] mimetypes = ['text/rust'] keyword_types = ( -- cgit v1.2.1 From 76bf6c18c554bda6b77199ef3fb0570a9017453f Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 29 Jan 2017 08:32:20 +0100 Subject: Clojure: extend valid_name to contain "|". --- pygments/lexers/jvm.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pygments/lexers/jvm.py b/pygments/lexers/jvm.py index f4392839..ccff41c1 100644 --- a/pygments/lexers/jvm.py +++ b/pygments/lexers/jvm.py @@ -801,7 +801,7 @@ class ClojureLexer(RegexLexer): # TODO / should divide keywords/symbols into namespace/rest # but that's hard, so just pretend / is part of the name - valid_name = r'(?!#)[\w!$%*+<=>?/.#-]+' + valid_name = r'(?!#)[\w!$%*+<=>?/.#|-]+' tokens = { 'root': [ -- cgit v1.2.1 From 39a6495a2305444f3e4219635398c345cd106f49 Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 29 Jan 2017 08:35:40 +0100 Subject: SCSS: fix wrong token type for single-quote strings (fixes #1322). --- pygments/lexers/css.py | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/pygments/lexers/css.py b/pygments/lexers/css.py index 29d83707..ce97730e 100644 --- a/pygments/lexers/css.py +++ b/pygments/lexers/css.py @@ -125,7 +125,7 @@ _css_properties = ( 'wrap-flow', 'wrap-inside', 'wrap-through', 'writing-mode', 'z-index', ) -# List of keyword values obtained from: +# List of keyword values obtained from: # http://cssvalues.com/ _keyword_values = ( 'absolute', 'alias', 'all', 'all-petite-caps', 'all-scroll', @@ -263,7 +263,7 @@ _time_units = ( 's', 'ms', ) _all_units = _angle_units + _frequency_units + _length_units + \ - _resolution_units + _time_units + _resolution_units + _time_units class CssLexer(RegexLexer): @@ -322,16 +322,18 @@ class CssLexer(RegexLexer): include('urls'), (r'('+r'|'.join(_functional_notation_keyword_values)+r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (words(_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_other_keyword_values, suffix=r'\b'), Keyword.Constant), (words(_color_keywords, suffix=r'\b'), Keyword.Constant), - (words(_css_properties, suffix=r'\b'), Keyword), # for transition-property etc. + # for transition-property etc. + (words(_css_properties, suffix=r'\b'), Keyword), (r'\!important', Comment.Preproc), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), - + (r'[~^*!%&<>|+=@:./?-]+', Operator), (r'[\[\](),]+', Punctuation), (r'"(\\\\|\\"|[^"])*"', String.Double), @@ -351,7 +353,8 @@ class CssLexer(RegexLexer): # function-start may be entered recursively (r'(' + r'|'.join(_functional_notation_keyword_values) + r')(\()', bygroups(Name.Builtin, Punctuation), 'function-start'), - (r'([a-zA-Z_][\w-]+)(\()', bygroups(Name.Function, Punctuation), 'function-start'), + (r'([a-zA-Z_][\w-]+)(\()', + bygroups(Name.Function, Punctuation), 'function-start'), (r'/\*(?:.|\n)*?\*/', Comment), include('numeric-values'), @@ -373,8 +376,8 @@ class CssLexer(RegexLexer): 'numeric-values': [ (r'\#[a-zA-Z0-9]{1,6}', Number.Hex), (r'[+\-]?[0-9]*[.][0-9]+', Number.Float, 'numeric-end'), - (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), - ], + (r'[+\-]?[0-9]+', Number.Integer, 'numeric-end'), + ], 'numeric-end': [ (words(_all_units, suffix=r'\b'), Keyword.Type), (r'%', Keyword.Type), @@ -466,9 +469,9 @@ common_sass_tokens = { ], 'string-single': [ - (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Double), + (r"(\\.|#(?=[^\n{])|[^\n'#])+", String.Single), (r'#\{', String.Interpol, 'interpolation'), - (r"'", String.Double, '#pop'), + (r"'", String.Single, '#pop'), ], 'string-url': [ -- cgit v1.2.1 From 968b078eb424095305a53470760a68d94cfb629b Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sun, 5 Feb 2017 17:17:05 +0100 Subject: AMPL: fix markup --- pygments/lexers/ampl.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pygments/lexers/ampl.py b/pygments/lexers/ampl.py index d439cb19..638d025d 100644 --- a/pygments/lexers/ampl.py +++ b/pygments/lexers/ampl.py @@ -3,7 +3,7 @@ pygments.lexers.ampl ~~~~~~~~~~~~~~~~~~~~ - Lexers for the ampl language. + Lexers for the AMPL language. :copyright: Copyright 2006-2017 by the Pygments team, see AUTHORS. :license: BSD, see LICENSE for details. @@ -18,7 +18,7 @@ __all__ = ['AmplLexer'] class AmplLexer(RegexLexer): """ - For AMPL source code. + For `AMPL `_ source code. .. versionadded:: 2.2 """ -- cgit v1.2.1 From 4f5f39dcfe35c329aa2d1be65724782092dd783e Mon Sep 17 00:00:00 2001 From: Erkki Mattila Date: Sun, 5 Feb 2017 17:52:37 +0200 Subject: Change instances of Comment.Singleline to Comment.Single --- pygments/lexers/asm.py | 2 +- pygments/lexers/automation.py | 4 ++-- pygments/lexers/c_like.py | 2 +- pygments/lexers/idl.py | 2 +- pygments/lexers/perl.py | 4 ++-- 5 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pygments/lexers/asm.py b/pygments/lexers/asm.py index 9c58478e..5b8cab80 100644 --- a/pygments/lexers/asm.py +++ b/pygments/lexers/asm.py @@ -265,7 +265,7 @@ class HsailLexer(RegexLexer): ], 'comments': [ (r'/\*.*?\*/', Comment.Multiline), - (r'//.*?\n', Comment.Singleline), + (r'//.*?\n', Comment.Single), ], 'keyword': [ # Types diff --git a/pygments/lexers/automation.py b/pygments/lexers/automation.py index be1ec129..3ef42e48 100644 --- a/pygments/lexers/automation.py +++ b/pygments/lexers/automation.py @@ -31,8 +31,8 @@ class AutohotkeyLexer(RegexLexer): 'root': [ (r'^(\s*)(/\*)', bygroups(Text, Comment.Multiline), 'incomment'), (r'^(\s*)(\()', bygroups(Text, Generic), 'incontinuation'), - (r'\s+;.*?$', Comment.Singleline), - (r'^;.*?$', Comment.Singleline), + (r'\s+;.*?$', Comment.Single), + (r'^;.*?$', Comment.Single), (r'[]{}(),;[]', Punctuation), (r'(in|is|and|or|not)\b', Operator.Word), (r'\%[a-zA-Z_#@$][\w#@$]*\%', Name.Variable), diff --git a/pygments/lexers/c_like.py b/pygments/lexers/c_like.py index f7ba7e8f..38827219 100644 --- a/pygments/lexers/c_like.py +++ b/pygments/lexers/c_like.py @@ -105,7 +105,7 @@ class ClayLexer(RegexLexer): tokens = { 'root': [ (r'\s', Text), - (r'//.*?$', Comment.Singleline), + (r'//.*?$', Comment.Single), (r'/(\\\n)?[*](.|\n)*?[*](\\\n)?/', Comment.Multiline), (r'\b(public|private|import|as|record|variant|instance' r'|define|overload|default|external|alias' diff --git a/pygments/lexers/idl.py b/pygments/lexers/idl.py index 99078970..2fc39318 100644 --- a/pygments/lexers/idl.py +++ b/pygments/lexers/idl.py @@ -249,7 +249,7 @@ class IDLLexer(RegexLexer): tokens = { 'root': [ - (r'^\s*;.*?\n', Comment.Singleline), + (r'^\s*;.*?\n', Comment.Single), (words(_RESERVED, prefix=r'\b', suffix=r'\b'), Keyword), (words(_BUILTIN_LIB, prefix=r'\b', suffix=r'\b'), Name.Builtin), (r'\+=|-=|\^=|\*=|/=|#=|##=|<=|>=|=', Operator), diff --git a/pygments/lexers/perl.py b/pygments/lexers/perl.py index 4d5ab3b3..db5a9361 100644 --- a/pygments/lexers/perl.py +++ b/pygments/lexers/perl.py @@ -489,7 +489,7 @@ class Perl6Lexer(ExtendedRegexLexer): 'common': [ (r'#[`|=](?P(?P[' + ''.join(PERL6_BRACKETS) + r'])(?P=first_char)*)', brackets_callback(Comment.Multiline)), - (r'#[^\n]*$', Comment.Singleline), + (r'#[^\n]*$', Comment.Single), (r'^(\s*)=begin\s+(\w+)\b.*?^\1=end\s+\2', Comment.Multiline), (r'^(\s*)=for.*?\n\s*?\n', Comment.Multiline), (r'^=.*?\n\s*?\n', Comment.Multiline), @@ -558,7 +558,7 @@ class Perl6Lexer(ExtendedRegexLexer): # make sure that '#' characters in quotes aren't treated as comments (r"(? Date: Tue, 28 Mar 2017 11:21:29 +0200 Subject: Update Dockerfile lexer Add Dockerfiles new keyword in the lexer Improve lexer to parse json arrays on specific commands Handle line breaks as a bash syntax --- pygments/lexers/configs.py | 22 ++++++++++++++-------- tests/examplefiles/docker.docker | 33 +++++++++++++++++++++++++++++++-- 2 files changed, 45 insertions(+), 10 deletions(-) diff --git a/pygments/lexers/configs.py b/pygments/lexers/configs.py index c39b1a52..4af2adb6 100644 --- a/pygments/lexers/configs.py +++ b/pygments/lexers/configs.py @@ -15,6 +15,7 @@ from pygments.lexer import RegexLexer, default, words, bygroups, include, using from pygments.token import Text, Comment, Operator, Keyword, Name, String, \ Number, Punctuation, Whitespace, Literal from pygments.lexers.shell import BashLexer +from pygments.lexers.data import JsonLexer __all__ = ['IniLexer', 'RegeditLexer', 'PropertiesLexer', 'KconfigLexer', 'Cfengine3Lexer', 'ApacheConfLexer', 'SquidConfLexer', @@ -539,20 +540,25 @@ class DockerLexer(RegexLexer): filenames = ['Dockerfile', '*.docker'] mimetypes = ['text/x-dockerfile-config'] - _keywords = (r'(?:FROM|MAINTAINER|CMD|EXPOSE|ENV|ADD|ENTRYPOINT|' - r'VOLUME|WORKDIR)') - + _keywords = (r'(?:FROM|MAINTAINER|EXPOSE|WORKDIR|USER|STOPSIGNAL)') + _bash_keywords = (r'(?:RUN|CMD|ENTRYPOINT|ENV|ARG|LABEL|ADD|COPY)') + _lb = r'(?:\s*\\?\s*)' # dockerfile line break regex flags = re.IGNORECASE | re.MULTILINE tokens = { 'root': [ - (r'^(ONBUILD)(\s+)(%s)\b' % (_keywords,), - bygroups(Name.Keyword, Whitespace, Keyword)), - (r'^(%s)\b(.*)' % (_keywords,), bygroups(Keyword, String)), (r'#.*', Comment), - (r'RUN', Keyword), # Rest of line falls through + (r'(ONBUILD)(%s)' % (_lb,), bygroups(Keyword, using(BashLexer))), + (r'(HEALTHCHECK)((%s--\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(VOLUME|ENTRYPOINT|CMD|SHELL)(%s)(\[.*?\])' % (_lb,), + bygroups(Keyword, using(BashLexer), using(JsonLexer))), + (r'(LABEL|ENV|ARG)((%s\w+=\w+%s)*)' % (_lb, _lb), + bygroups(Keyword, using(BashLexer))), + (r'(%s|VOLUME)\b(.*)' % (_keywords), bygroups(Keyword, String)), + (r'(%s)' % (_bash_keywords,), Keyword), (r'(.*\\\n)*.+', using(BashLexer)), - ], + ] } diff --git a/tests/examplefiles/docker.docker b/tests/examplefiles/docker.docker index d65385b6..1ae3c3a1 100644 --- a/tests/examplefiles/docker.docker +++ b/tests/examplefiles/docker.docker @@ -1,5 +1,34 @@ -maintainer First O'Last +FROM alpine:3.5 +MAINTAINER First O'Last +# comment run echo \ 123 $bar -# comment +RUN apk --update add rsync dumb-init + +# Test env with both syntax +ENV FOO = "BAR" +ENV FOO \ + "BAR" + +COPY foo "bar" +COPY foo \ + "bar" + +HEALTHCHECK \ + --interval=5m --timeout=3s \ + CMD curl -f http://localhost/ || exit 1 + +# ONBUILD keyword, then with linebreak +ONBUILD ADD . /app/src +ONBUILD \ + RUN echo 123 $bar + +# Potential JSON array parsing, mixed with linebreaks +VOLUME \ + /foo +VOLUME \ + ["/bar"] +VOLUME ["/bar"] +VOLUME /foo +CMD ["foo", "bar"] -- cgit v1.2.1 From 5927c118d3556f19199fe502c0ffcc8584652518 Mon Sep 17 00:00:00 2001 From: Miro Hron?ok Date: Wed, 4 Jul 2018 18:08:48 +0200 Subject: PEP 479: Raising StopIteration from a generator is now an error So we return instead. Fix needed for Python 3.7. Fixes https://bitbucket.org/birkenfeld/pygments-main/issues/1457 --- pygments/lexers/lisp.py | 4 ++-- pygments/lexers/sql.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..258916df 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -2327,13 +2327,13 @@ class ShenLexer(RegexLexer): token = Name.Function if token == Literal else token yield index, token, value - raise StopIteration + return def _process_signature(self, tokens): for index, token, value in tokens: if token == Literal and value == '}': yield index, Punctuation, value - raise StopIteration + return elif token in (Literal, Name.Function): token = Name.Variable if value.istitle() else Keyword.Type yield index, token, value diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 7507c0fc..05af51ba 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -347,7 +347,10 @@ class PostgresConsoleLexer(Lexer): # Emit the output lines out_token = Generic.Output while 1: - line = next(lines) + try: + line = next(lines) + except StopIteration: + return mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt -- cgit v1.2.1 From b909c279a4cf4a93c5244e2c3046d769ba737dc0 Mon Sep 17 00:00:00 2001 From: Miro Hron?ok Date: Wed, 4 Jul 2018 18:31:40 +0200 Subject: Be more Pythonic, use a for instead of while 1: try: next --- pygments/lexers/sql.py | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/pygments/lexers/sql.py b/pygments/lexers/sql.py index 05af51ba..3f7dfdb8 100644 --- a/pygments/lexers/sql.py +++ b/pygments/lexers/sql.py @@ -308,14 +308,7 @@ class PostgresConsoleLexer(Lexer): # and continue until the end of command is detected curcode = '' insertions = [] - while 1: - try: - line = next(lines) - except StopIteration: - # allow the emission of partially collected items - # the repl loop will be broken below - break - + for line in lines: # Identify a shell prompt in case of psql commandline example if line.startswith('$') and not curcode: lexer = get_lexer_by_name('console', **self.options) @@ -346,11 +339,7 @@ class PostgresConsoleLexer(Lexer): # Emit the output lines out_token = Generic.Output - while 1: - try: - line = next(lines) - except StopIteration: - return + for line in lines: mprompt = re_prompt.match(line) if mprompt is not None: # push the line back to have it processed by the prompt @@ -366,6 +355,8 @@ class PostgresConsoleLexer(Lexer): yield (mmsg.start(2), out_token, mmsg.group(2)) else: yield (0, out_token, line) + else: + return class SqlLexer(RegexLexer): -- cgit v1.2.1 From 0342044ec7dcd26d6ab911c87977aa5055bb3336 Mon Sep 17 00:00:00 2001 From: Edward Betts Date: Sun, 4 Nov 2018 07:51:02 +0000 Subject: Use print as a function so the example works in Python 3. --- doc/docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docs/quickstart.rst b/doc/docs/quickstart.rst index dba7698a..d994f74e 100644 --- a/doc/docs/quickstart.rst +++ b/doc/docs/quickstart.rst @@ -39,7 +39,7 @@ Here is a small example for highlighting Python code: from pygments.formatters import HtmlFormatter code = 'print "Hello World"' - print highlight(code, PythonLexer(), HtmlFormatter()) + print(highlight(code, PythonLexer(), HtmlFormatter())) which prints something like this: -- cgit v1.2.1 From c1255470b48207d060454abcf2a0b3709db5904b Mon Sep 17 00:00:00 2001 From: Phil Hagelberg Date: Fri, 16 Nov 2018 18:11:03 -0800 Subject: Add support for the Fennel programming language This is a pretty straightforward language in the lisp family with a small number of special forms. Since Fennel runs on the Lua runtime, the list of builtins is the same as that of Lua, so it might be possible to re-use the definition from the Lua lexer, but since I don't know Python I couldn't figure out how that would work; maybe someone else could add that. --- pygments/lexers/_mapping.py | 1 + pygments/lexers/lisp.py | 71 ++++++++++++++++- tests/examplefiles/fennelview.fnl | 156 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 227 insertions(+), 1 deletion(-) create mode 100644 tests/examplefiles/fennelview.fnl diff --git a/pygments/lexers/_mapping.py b/pygments/lexers/_mapping.py index b48ee1d1..6ed0852d 100644 --- a/pygments/lexers/_mapping.py +++ b/pygments/lexers/_mapping.py @@ -153,6 +153,7 @@ LEXERS = { 'FancyLexer': ('pygments.lexers.ruby', 'Fancy', ('fancy', 'fy'), ('*.fy', '*.fancypack'), ('text/x-fancysrc',)), 'FantomLexer': ('pygments.lexers.fantom', 'Fantom', ('fan',), ('*.fan',), ('application/x-fantom',)), 'FelixLexer': ('pygments.lexers.felix', 'Felix', ('felix', 'flx'), ('*.flx', '*.flxh'), ('text/x-felix',)), + 'FennelLexer': ('pygments.lexers.lisp', 'Fennel', (), ('*.fnl',), ()), 'FishShellLexer': ('pygments.lexers.shell', 'Fish', ('fish', 'fishshell'), ('*.fish', '*.load'), ('application/x-fish',)), 'FlatlineLexer': ('pygments.lexers.dsls', 'Flatline', ('flatline',), (), ('text/x-flatline',)), 'ForthLexer': ('pygments.lexers.forth', 'Forth', ('forth',), ('*.frt', '*.fs'), ('application/x-forth',)), diff --git a/pygments/lexers/lisp.py b/pygments/lexers/lisp.py index e258c347..345dabfb 100644 --- a/pygments/lexers/lisp.py +++ b/pygments/lexers/lisp.py @@ -19,7 +19,7 @@ from pygments.lexers.python import PythonLexer __all__ = ['SchemeLexer', 'CommonLispLexer', 'HyLexer', 'RacketLexer', 'NewLispLexer', 'EmacsLispLexer', 'ShenLexer', 'CPSALexer', - 'XtlangLexer'] + 'XtlangLexer', 'FennelLexer'] class SchemeLexer(RegexLexer): @@ -2619,3 +2619,72 @@ class XtlangLexer(RegexLexer): include('scheme') ], } + +class FennelLexer(RegexLexer): + """A lexer for the Fennel programming language + + Fennel compiles to Lua, so all the Lua builtins are recognized as well + as the special forms that are particular to the Fennel compiler. + """ + name = 'Fennel' + aliases = ['fennel', 'fnl'] + filenames = ['*.fnl'] + + # these two lists are taken from fennel-mode.el: + # https://gitlab.com/technomancy/fennel-mode + # this list is current as of Fennel version 0.1.0. + special_forms = ( + u'require-macros', u'eval-compiler', + u'do', u'values', u'if', u'when', u'each', u'for', u'fn', u'lambda', + u'λ', u'set', u'global', u'var', u'local', u'let', u'tset', u'doto', + u'set-forcibly!', u'defn', u'partial', u'while', u'or', u'and', u'true', + u'false', u'nil', u'.', u'+', u'..', u'^', u'-', u'*', u'%', u'/', u'>', + u'<', u'>=', u'<=', u'=', u'~=', u'#', u'...', u':', u'->', u'->>', + ) + + # Might be nicer to use the list from _lua_builtins.py but it's unclear how? + builtins = ( + u'_G', u'_VERSION', u'arg', u'assert', u'bit32', u'collectgarbage', + u'coroutine', u'debug', u'dofile', u'error', u'getfenv', + u'getmetatable', u'io', u'ipairs', u'load', u'loadfile', u'loadstring', + u'math', u'next', u'os', u'package', u'pairs', u'pcall', u'print', + u'rawequal', u'rawget', u'rawlen', u'rawset', u'require', u'select', + u'setfenv', u'setmetatable', u'string', u'table', u'tonumber', + u'tostring', u'type', u'unpack', u'xpcall' + ) + + # based on the scheme definition, but disallowing leading digits and commas + valid_name = r'[a-zA-Z_!$%&*+/:<=>?@^~|-][\w!$%&*+/:<=>?@^~|\.-]*' + + tokens = { + 'root': [ + # the only comment form is a semicolon; goes to the end of the line + (r';.*$', Comment.Single), + + (r'[,\s]+', Text), + (r'-?\d+\.\d+', Number.Float), + (r'-?\d+', Number.Integer), + + (r'"(\\\\|\\"|[^"])*"', String), + (r"'(\\\\|\\'|[^'])*'", String), + + # these are technically strings, but it's worth visually + # distinguishing them because their intent is different + # from regular strings. + (r':' + valid_name, String.Symbol), + + # special forms are keywords + (words(special_forms, suffix=' '), Keyword), + # lua standard library are builtins + (words(builtins, suffix=' '), Name.Builtin), + # special-case the vararg symbol + (r'\.\.\.', Name.Variable), + # regular identifiers + (valid_name, Name.Variable), + + # all your normal paired delimiters for your programming enjoyment + (r'(\(|\))', Punctuation), + (r'(\[|\])', Punctuation), + (r'(\{|\})', Punctuation), + ] + } diff --git a/tests/examplefiles/fennelview.fnl b/tests/examplefiles/fennelview.fnl new file mode 100644 index 00000000..fd0fc648 --- /dev/null +++ b/tests/examplefiles/fennelview.fnl @@ -0,0 +1,156 @@ +;; A pretty-printer that outputs tables in Fennel syntax. +;; Loosely based on inspect.lua: http://github.com/kikito/inspect.lua + +(local quote (fn [str] (.. '"' (: str :gsub '"' '\\"') '"'))) + +(local short-control-char-escapes + {"\a" "\\a" "\b" "\\b" "\f" "\\f" "\n" "\\n" + "\r" "\\r" "\t" "\\t" "\v" "\\v"}) + +(local long-control-char-esapes + (let [long {}] + (for [i 0 31] + (let [ch (string.char i)] + (when (not (. short-control-char-escapes ch)) + (tset short-control-char-escapes ch (.. "\\" i)) + (tset long ch (: "\\%03d" :format i))))) + long)) + +(fn escape [str] + (let [str (: str :gsub "\\" "\\\\") + str (: str :gsub "(%c)%f[0-9]" long-control-char-esapes)] + (: str :gsub "%c" short-control-char-escapes))) + +(fn sequence-key? [k len] + (and (= (type k) "number") + (<= 1 k) + (<= k len) + (= (math.floor k) k))) + +(local type-order {:number 1 :boolean 2 :string 3 :table 4 + :function 5 :userdata 6 :thread 7}) + +(fn sort-keys [a b] + (let [ta (type a) tb (type b)] + (if (and (= ta tb) (~= ta "boolean") + (or (= ta "string") (= ta "number"))) + (< a b) + (let [dta (. type-order a) + dtb (. type-order b)] + (if (and dta dtb) + (< dta dtb) + dta true + dtb false + :else (< ta tb)))))) + +(fn get-sequence-length [t] + (var len 1) + (each [i (ipairs t)] (set len i)) + len) + +(fn get-nonsequential-keys [t] + (let [keys {} + sequence-length (get-sequence-length t)] + (each [k (pairs t)] + (when (not (sequence-key? k sequence-length)) + (table.insert keys k))) + (table.sort keys sort-keys) + (values keys sequence-length))) + +(fn count-table-appearances [t appearances] + (if (= (type t) "table") + (when (not (. appearances t)) + (tset appearances t 1) + (each [k v (pairs t)] + (count-table-appearances k appearances) + (count-table-appearances v appearances))) + (when (and t (= t t)) ; no nans please + (tset appearances t (+ (or (. appearances t) 0) 1)))) + appearances) + + + +(var put-value nil) ; mutual recursion going on; defined below + +(fn puts [self ...] + (each [_ v (ipairs [...])] + (table.insert self.buffer v))) + +(fn tabify [self] (puts self "\n" (: self.indent :rep self.level))) + +(fn already-visited? [self v] (~= (. self.ids v) nil)) + +(fn get-id [self v] + (var id (. self.ids v)) + (when (not id) + (let [tv (type v)] + (set id (+ (or (. self.max-ids tv) 0) 1)) + (tset self.max-ids tv id) + (tset self.ids v id))) + (tostring id)) + +(fn put-sequential-table [self t length] + (puts self "[") + (set self.level (+ self.level 1)) + (for [i 1 length] + (puts self " ") + (put-value self (. t i))) + (set self.level (- self.level 1)) + (puts self " ]")) + +(fn put-key [self k] + (if (and (= (type k) "string") + (: k :find "^[-%w?\\^_`!#$%&*+./@~:|<=>]+$")) + (puts self ":" k) + (put-value self k))) + +(fn put-kv-table [self t] + (puts self "{") + (set self.level (+ self.level 1)) + (each [k v (pairs t)] + (tabify self) + (put-key self k) + (puts self " ") + (put-value self v)) + (set self.level (- self.level 1)) + (tabify self) + (puts self "}")) + +(fn put-table [self t] + (if (already-visited? self t) + (puts self "#") + (>= self.level self.depth) + (puts self "{...}") + :else + (let [(non-seq-keys length) (get-nonsequential-keys t) + id (get-id self t)] + (if (> (. self.appearances t) 1) + (puts self "#<" id ">") + (and (= (# non-seq-keys) 0) (= (# t) 0)) + (puts self "{}") + (= (# non-seq-keys) 0) + (put-sequential-table self t length) + :else + (put-kv-table self t))))) + +(set put-value (fn [self v] + (let [tv (type v)] + (if (= tv "string") + (puts self (quote (escape v))) + (or (= tv "number") (= tv "boolean") (= tv "nil")) + (puts self (tostring v)) + (= tv "table") + (put-table self v) + :else + (puts self "#<" (tostring v) ">"))))) + + + +(fn fennelview [root options] + (let [options (or options {}) + inspector {:appearances (count-table-appearances root {}) + :depth (or options.depth 128) + :level 0 :buffer {} :ids {} :max-ids {} + :indent (or options.indent " ")}] + (put-value inspector root) + (table.concat inspector.buffer))) -- cgit v1.2.1 From bd2cb3c85d549bee2bcc6e4fa882470b4acbda5d Mon Sep 17 00:00:00 2001 From: Georg Brandl Date: Sat, 24 Nov 2018 15:26:06 +0100 Subject: add release checklist --- scripts/release-checklist | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 scripts/release-checklist diff --git a/scripts/release-checklist b/scripts/release-checklist new file mode 100644 index 00000000..f18e6376 --- /dev/null +++ b/scripts/release-checklist @@ -0,0 +1,24 @@ +Release checklist +================= + +* Check hg status +* Make check +* Make pylint +* Make test from clean checkout with all supported Python versions +* Update ez_setup.py +* Update version info in setup.py/__init__.py +* Check setup.py metadata: long description, trove classifiers +* Update release date/code name in CHANGES +* hg commit +* make clean +* For every supported version: + pythonX.Y setup.py release bdist_egg sdist upload +* Check PyPI release page for obvious errors +* hg tag +* Make a maintenance branch if applicable +* Update homepage (release info), regenerate docs (+printable!) +* Add new version/milestone to tracker categories +* Write announcement and send to mailing list/python-announce +* Update version info, add new CHANGES entry for next version +* hg commit +* hg push -- cgit v1.2.1 From 60afd6c7897f2e0b5326645efd48c9b70db744d8 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:49:57 +0100 Subject: Fix the tests, and enable them for Python 2.7, 3.5, 3.6, 3.7. Updated the documentation as well to make clear which Python versions are officially supported. --- doc/faq.rst | 4 ++-- pygments/lexers/xorg.py | 4 ++-- tox.ini | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/doc/faq.rst b/doc/faq.rst index f375828b..172929e0 100644 --- a/doc/faq.rst +++ b/doc/faq.rst @@ -35,8 +35,8 @@ and in this case, source code! What are the system requirements? --------------------------------- -Pygments only needs a standard Python install, version 2.6 or higher or version -3.3 or higher for Python 3. No additional libraries are needed. +Pygments only needs a standard Python install, version 2.7 or higher or version +3.5 or higher for Python 3. No additional libraries are needed. How can I use Pygments? ----------------------- diff --git a/pygments/lexers/xorg.py b/pygments/lexers/xorg.py index 89475a80..e6383b30 100644 --- a/pygments/lexers/xorg.py +++ b/pygments/lexers/xorg.py @@ -26,8 +26,8 @@ class XorgLexer(RegexLexer): (r'\s+', Text), (r'#.*$', Comment), - (r'((|Sub)Section)(\s+)("\w+")', - bygroups(String.Escape, String.Escape, Text, String.Escape)), + (r'((?:Sub)?Section)(\s+)("\w+")', + bygroups(String.Escape, Text, String.Escape)), (r'(End(|Sub)Section)', String.Escape), (r'(\w+)(\s+)([^\n#]+)', diff --git a/tox.ini b/tox.ini index 8a33f99c..2c63c292 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py26, py27, py33, py34 +envlist = py27, py35, py36, py37 [testenv] deps = nose -- cgit v1.2.1 From 72b98953d2dd8db2a909595eea718744ed7ee318 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:50:46 +0100 Subject: Fix another instance of print in the quickstart documentation. --- doc/docs/quickstart.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/docs/quickstart.rst b/doc/docs/quickstart.rst index d994f74e..3a823e7f 100644 --- a/doc/docs/quickstart.rst +++ b/doc/docs/quickstart.rst @@ -56,7 +56,7 @@ can be produced by: .. sourcecode:: python - print HtmlFormatter().get_style_defs('.highlight') + print(HtmlFormatter().get_style_defs('.highlight')) The argument to :func:`get_style_defs` is used as an additional CSS selector: the output may look like this: -- cgit v1.2.1 From 6bc8c0de737b181566fa66bc29f5fef4bc6ab11c Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:52:06 +0100 Subject: Fix documentation not building with latest Sphinx. --- pygments/lexers/xorg.py | 1 + pygments/sphinxext.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pygments/lexers/xorg.py b/pygments/lexers/xorg.py index e6383b30..3bba930f 100644 --- a/pygments/lexers/xorg.py +++ b/pygments/lexers/xorg.py @@ -16,6 +16,7 @@ __all__ = ['XorgLexer'] class XorgLexer(RegexLexer): + """Lexer for xorg.conf file.""" name = 'Xorg' aliases = ['xorg.conf'] filenames = ['xorg.conf'] diff --git a/pygments/sphinxext.py b/pygments/sphinxext.py index f962f8c6..5c8ac8b9 100644 --- a/pygments/sphinxext.py +++ b/pygments/sphinxext.py @@ -16,7 +16,7 @@ import sys from docutils import nodes from docutils.statemachine import ViewList -from sphinx.util.compat import Directive +from docutils.parsers.rst import Directive from sphinx.util.nodes import nested_parse_with_titles -- cgit v1.2.1 From aa765094469cb90eff740b46a7c5c5903204c098 Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:55:37 +0100 Subject: Add Fennel and HLSL to the list of supported languages. --- AUTHORS | 3 ++- doc/languages.rst | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/AUTHORS b/AUTHORS index f39bbaa1..18e642f1 100644 --- a/AUTHORS +++ b/AUTHORS @@ -72,7 +72,7 @@ Other contributors, listed alphabetically, are: * Alex Gosse -- TrafficScript lexer * Patrick Gotthardt -- PHP namespaces support * Olivier Guibe -- Asymptote lexer -* Jordi Gutiérrez Hermoso -- Octave lexer +* Phil Hagelberg -- Fennel lexer * Florian Hahn -- Boogie lexer * Martin Harriman -- SNOBOL lexer * Matthew Harrison -- SVG formatter @@ -81,6 +81,7 @@ Other contributors, listed alphabetically, are: * Aslak Hellesøy -- Gherkin lexer * Greg Hendershott -- Racket lexer * Justin Hendrick -- ParaSail lexer +* Jordi Gutiérrez Hermoso -- Octave lexer * David Hess, Fish Software, Inc. -- Objective-J lexer * Varun Hiremath -- Debian control lexer * Rob Hoelz -- Perl 6 lexer diff --git a/doc/languages.rst b/doc/languages.rst index 7fa8eb2f..e5399403 100644 --- a/doc/languages.rst +++ b/doc/languages.rst @@ -37,6 +37,7 @@ Programming languages * `Ezhil `_ Ezhil - A Tamil programming language * Factor * Fancy +* `Fennel `_ * Fortran * F# * GAP @@ -44,6 +45,7 @@ Programming languages * GL shaders * Groovy * `Haskell `_ (incl. Literate Haskell) +* HLSL * IDL * Io * Java -- cgit v1.2.1 From a37f5614f11f3d5af660a024f1749dc7eee7e8ad Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 17:56:44 +0100 Subject: Add Bitbucket CI configuration. --- bitbucket-pipelines.yml | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 bitbucket-pipelines.yml diff --git a/bitbucket-pipelines.yml b/bitbucket-pipelines.yml new file mode 100644 index 00000000..e3c74d6e --- /dev/null +++ b/bitbucket-pipelines.yml @@ -0,0 +1,34 @@ +pipelines: + default: + - step: + name: Test on Python 2.7 + image: python:2.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py27 + - step: + name: Test on Python 3.4 + image: python:3.4 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py34 + - step: + name: Test on Python 3.6 + image: python:3.6 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py36 + - step: + name: Test on Python 3.7 + image: python:3.7 + caches: + - pip + script: + - pip install -r requirements.txt + - tox -e py37 \ No newline at end of file -- cgit v1.2.1 From 4792cccf7886199bd6d49dbe487f213c0f8094ce Mon Sep 17 00:00:00 2001 From: "Matth?us G. Chajdas" Date: Sat, 24 Nov 2018 18:04:30 +0100 Subject: Update changelog and version number. --- CHANGES | 13 +++++++++++++ pygments/__init__.py | 2 +- setup.py | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGES b/CHANGES index 0bab9118..af4141f2 100644 --- a/CHANGES +++ b/CHANGES @@ -6,6 +6,19 @@ Issue numbers refer to the tracker at pull request numbers to the requests at . +Version 2.3.0 +------------- +(not yet released) + +- Added lexers: + + * Fennel (PR#783) + * HLSL (PR#675) + +- Added Python 3.7 support (PR#772) +- Fix incorrect token type in SCSS for single-quote strings (#1322) +- Use `terminal256` formatter if `TERM` contains `256` (PR#666) + Version 2.2.0 ------------- (release Jan 22, 2017) diff --git a/pygments/__init__.py b/pygments/__init__.py index 394a85f2..19aafdeb 100644 --- a/pygments/__init__.py +++ b/pygments/__init__.py @@ -29,7 +29,7 @@ import sys from pygments.util import StringIO, BytesIO -__version__ = '2.2.0' +__version__ = '2.3.0' __docformat__ = 'restructuredtext' __all__ = ['lex', 'format', 'highlight'] diff --git a/setup.py b/setup.py index 1705923c..7e6eca8c 100755 --- a/setup.py +++ b/setup.py @@ -48,7 +48,7 @@ else: setup( name = 'Pygments', - version = '2.2.0', + version = '2.3.0', url = 'http://pygments.org/', license = 'BSD License', author = 'Georg Brandl', -- cgit v1.2.1