diff options
Diffstat (limited to 'lib/profile')
23 files changed, 323 insertions, 153 deletions
diff --git a/lib/profile/CMakeLists.txt b/lib/profile/CMakeLists.txt index 488673dd2..9774be6a8 100644 --- a/lib/profile/CMakeLists.txt +++ b/lib/profile/CMakeLists.txt @@ -62,6 +62,7 @@ set(PROFILE_SOURCES InstrProfilingPlatformFuchsia.c InstrProfilingPlatformLinux.c InstrProfilingPlatformOther.c + InstrProfilingPlatformWindows.c InstrProfilingRuntime.cc InstrProfilingUtil.c) diff --git a/lib/profile/GCDAProfiling.c b/lib/profile/GCDAProfiling.c index 0665a680c..498c05900 100644 --- a/lib/profile/GCDAProfiling.c +++ b/lib/profile/GCDAProfiling.c @@ -1,9 +1,8 @@ /*===- GCDAProfiling.c - Support library for GCDA file emission -----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------===*| |* @@ -269,14 +268,14 @@ static int map_file() { mmap_handle = CreateFileMapping(mmap_fd, NULL, PAGE_READWRITE, DWORD_HI(file_size), DWORD_LO(file_size), NULL); if (mmap_handle == NULL) { - fprintf(stderr, "profiling: %s: cannot create file mapping: %d\n", filename, - GetLastError()); + fprintf(stderr, "profiling: %s: cannot create file mapping: %lu\n", + filename, GetLastError()); return -1; } write_buffer = MapViewOfFile(mmap_handle, FILE_MAP_WRITE, 0, 0, file_size); if (write_buffer == NULL) { - fprintf(stderr, "profiling: %s: cannot map: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot map: %lu\n", filename, GetLastError()); CloseHandle(mmap_handle); return -1; @@ -298,18 +297,18 @@ static int map_file() { static void unmap_file() { #if defined(_WIN32) if (!FlushViewOfFile(write_buffer, file_size)) { - fprintf(stderr, "profiling: %s: cannot flush mapped view: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot flush mapped view: %lu\n", filename, GetLastError()); } if (!UnmapViewOfFile(write_buffer)) { - fprintf(stderr, "profiling: %s: cannot unmap mapped view: %d\n", filename, + fprintf(stderr, "profiling: %s: cannot unmap mapped view: %lu\n", filename, GetLastError()); } if (!CloseHandle(mmap_handle)) { - fprintf(stderr, "profiling: %s: cannot close file mapping handle: %d\n", filename, - GetLastError()); + fprintf(stderr, "profiling: %s: cannot close file mapping handle: %lu\n", + filename, GetLastError()); } mmap_handle = NULL; diff --git a/lib/profile/InstrProfData.inc b/lib/profile/InstrProfData.inc index 454620ed9..749781b9a 100644 --- a/lib/profile/InstrProfData.inc +++ b/lib/profile/InstrProfData.inc @@ -1,9 +1,8 @@ /*===-- InstrProfData.inc - instr profiling runtime structures -*- C++ -*-=== *\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -170,7 +169,7 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) /* VALUE_PROF_KIND start */ #ifndef VALUE_PROF_KIND -#define VALUE_PROF_KIND(Enumerator, Value) +#define VALUE_PROF_KIND(Enumerator, Value, Descr) #else #define INSTR_PROF_DATA_DEFINED #endif @@ -183,16 +182,16 @@ VALUE_PROF_FUNC_PARAM(uint64_t, LargeValue, Type::getInt64Ty(Ctx)) * For this remapping the ProfData is used. ProfData contains both the function * name hash and the function address. */ -VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0) +VALUE_PROF_KIND(IPVK_IndirectCallTarget, 0, "indirect call target") /* For memory intrinsic functions size profiling. */ -VALUE_PROF_KIND(IPVK_MemOPSize, 1) +VALUE_PROF_KIND(IPVK_MemOPSize, 1, "memory intrinsic functions size") /* These two kinds must be the last to be * declared. This is to make sure the string * array created with the template can be * indexed with the kind value. */ -VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget) -VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize) +VALUE_PROF_KIND(IPVK_First, IPVK_IndirectCallTarget, "first") +VALUE_PROF_KIND(IPVK_Last, IPVK_MemOPSize, "last") #undef VALUE_PROF_KIND /* VALUE_PROF_KIND end */ @@ -250,22 +249,25 @@ COVMAP_HEADER(uint32_t, Int32Ty, Version, \ #define INSTR_PROF_DATA_DEFINED INSTR_PROF_SECT_ENTRY(IPSK_data, \ INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_COFF), "__DATA,") + INSTR_PROF_DATA_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_cnts, \ INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COFF), "__DATA,") + INSTR_PROF_CNTS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_name, \ INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_COFF), "__DATA,") + INSTR_PROF_NAME_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vals, \ INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_COFF), "__DATA,") + INSTR_PROF_VALS_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_vnodes, \ INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COFF), "__DATA,") + INSTR_PROF_VNODES_COFF, "__DATA,") INSTR_PROF_SECT_ENTRY(IPSK_covmap, \ INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON), \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COFF), "__LLVM_COV,") + INSTR_PROF_COVMAP_COFF, "__LLVM_COV,") +INSTR_PROF_SECT_ENTRY(IPSK_orderfile, \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON), \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COFF), "__DATA,") #undef INSTR_PROF_SECT_ENTRY #endif @@ -636,10 +638,12 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, * version for other variants of profile. We set the lowest bit of the upper 8 * bits (i.e. bit 56) to 1 to indicate if this is an IR-level instrumentaiton * generated profile, and 0 if this is a Clang FE generated profile. + * 1 in bit 57 indicates there are context-sensitive records in the profile. */ #define VARIANT_MASKS_ALL 0xff00000000000000ULL #define GET_VERSION(V) ((V) & ~VARIANT_MASKS_ALL) #define VARIANT_MASK_IR_PROF (0x1ULL << 56) +#define VARIANT_MASK_CSIR_PROF (0x1ULL << 57) #define INSTR_PROF_RAW_VERSION_VAR __llvm_profile_raw_version #define INSTR_PROF_PROFILE_RUNTIME_VAR __llvm_profile_runtime @@ -655,13 +659,17 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, #define INSTR_PROF_VALS_COMMON __llvm_prf_vals #define INSTR_PROF_VNODES_COMMON __llvm_prf_vnds #define INSTR_PROF_COVMAP_COMMON __llvm_covmap -/* Win32 */ -#define INSTR_PROF_DATA_COFF .lprfd -#define INSTR_PROF_NAME_COFF .lprfn -#define INSTR_PROF_CNTS_COFF .lprfc -#define INSTR_PROF_VALS_COFF .lprfv -#define INSTR_PROF_VNODES_COFF .lprfnd -#define INSTR_PROF_COVMAP_COFF .lcovmap +#define INSTR_PROF_ORDERFILE_COMMON __llvm_orderfile +/* Windows section names. Because these section names contain dollar characters, + * they must be quoted. + */ +#define INSTR_PROF_DATA_COFF ".lprfd$M" +#define INSTR_PROF_NAME_COFF ".lprfn$M" +#define INSTR_PROF_CNTS_COFF ".lprfc$M" +#define INSTR_PROF_VALS_COFF ".lprfv$M" +#define INSTR_PROF_VNODES_COFF ".lprfnd$M" +#define INSTR_PROF_COVMAP_COFF ".lcovmap$M" +#define INSTR_PROF_ORDERFILE_COFF ".lorderfile$M" #ifdef _WIN32 /* Runtime section names and name strings. */ @@ -675,32 +683,30 @@ serializeValueProfDataFrom(ValueProfRecordClosure *Closure, /* Value profile nodes section. */ #define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COFF #define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COFF +#define INSTR_PROF_ORDERFILE_SECT_NAME INSTR_PROF_ORDERFILE_COFF #else /* Runtime section names and name strings. */ -#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_DATA_COMMON -#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_NAME_COMMON -#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_CNTS_COMMON +#define INSTR_PROF_DATA_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_DATA_COMMON) +#define INSTR_PROF_NAME_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_NAME_COMMON) +#define INSTR_PROF_CNTS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_CNTS_COMMON) /* Array of pointers. Each pointer points to a list * of value nodes associated with one value site. */ -#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_VALS_COMMON +#define INSTR_PROF_VALS_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VALS_COMMON) /* Value profile nodes section. */ -#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_VNODES_COMMON -#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_COVMAP_COMMON +#define INSTR_PROF_VNODES_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_VNODES_COMMON) +#define INSTR_PROF_COVMAP_SECT_NAME INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_COMMON) +/* Order file instrumentation. */ +#define INSTR_PROF_ORDERFILE_SECT_NAME \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_COMMON) #endif -#define INSTR_PROF_DATA_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_DATA_SECT_NAME) -#define INSTR_PROF_NAME_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_NAME_SECT_NAME) -#define INSTR_PROF_CNTS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_CNTS_SECT_NAME) -#define INSTR_PROF_COVMAP_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_COVMAP_SECT_NAME) -#define INSTR_PROF_VALS_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VALS_SECT_NAME) -#define INSTR_PROF_VNODES_SECT_NAME_STR \ - INSTR_PROF_QUOTE(INSTR_PROF_VNODES_SECT_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_NAME _llvm_order_file_buffer +#define INSTR_PROF_ORDERFILE_BUFFER_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_NAME) +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME _llvm_order_file_buffer_idx +#define INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME_STR \ + INSTR_PROF_QUOTE(INSTR_PROF_ORDERFILE_BUFFER_IDX_NAME) /* Macros to define start/stop section symbol for a given * section on Linux. For instance @@ -735,6 +741,12 @@ typedef struct InstrProfValueData { #endif /* INSTR_PROF_DATA_INC */ +#ifndef INSTR_ORDER_FILE_INC +// The maximal # of functions: 128*1024 (the buffer size will be 128*4 KB). +#define INSTR_ORDER_FILE_BUFFER_SIZE 131072 +#define INSTR_ORDER_FILE_BUFFER_BITS 17 +#define INSTR_ORDER_FILE_BUFFER_MASK 0x1ffff +#endif /* INSTR_ORDER_FILE_INC */ #else #undef INSTR_PROF_DATA_DEFINED #endif diff --git a/lib/profile/InstrProfiling.c b/lib/profile/InstrProfiling.c index 00b31e1ee..299cf3177 100644 --- a/lib/profile/InstrProfiling.c +++ b/lib/profile/InstrProfiling.c @@ -1,9 +1,8 @@ /*===- InstrProfiling.c - Support library for PGO instrumentation ---------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfiling.h b/lib/profile/InstrProfiling.h index 3ef7eacf0..63aebaf79 100644 --- a/lib/profile/InstrProfiling.h +++ b/lib/profile/InstrProfiling.h @@ -1,9 +1,8 @@ /*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -16,7 +15,7 @@ #include "InstrProfData.inc" enum ValueKind { -#define VALUE_PROF_KIND(Enumerator, Value) Enumerator = Value, +#define VALUE_PROF_KIND(Enumerator, Value, Descr) Enumerator = Value, #include "InstrProfData.inc" }; @@ -65,6 +64,7 @@ uint64_t *__llvm_profile_begin_counters(void); uint64_t *__llvm_profile_end_counters(void); ValueProfNode *__llvm_profile_begin_vnodes(); ValueProfNode *__llvm_profile_end_vnodes(); +uint32_t *__llvm_profile_begin_orderfile(); /*! * \brief Clear profile counters to zero. @@ -121,6 +121,7 @@ void __llvm_profile_instrument_target_value(uint64_t TargetValue, void *Data, */ int __llvm_profile_write_file(void); +int __llvm_orderfile_write_file(void); /*! * \brief this is a wrapper interface to \c __llvm_profile_write_file. * After this interface is invoked, a arleady dumped flag will be set @@ -143,6 +144,8 @@ int __llvm_profile_write_file(void); */ int __llvm_profile_dump(void); +int __llvm_orderfile_dump(void); + /*! * \brief Set the filename for writing instrumentation data. * diff --git a/lib/profile/InstrProfilingBuffer.c b/lib/profile/InstrProfilingBuffer.c index a7e852f53..5bdeb8e32 100644 --- a/lib/profile/InstrProfilingBuffer.c +++ b/lib/profile/InstrProfilingBuffer.c @@ -1,9 +1,8 @@ /*===- InstrProfilingBuffer.c - Write instrumentation to a memory buffer --===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingFile.c b/lib/profile/InstrProfilingFile.c index c4cf3ccd7..8c32bde9d 100644 --- a/lib/profile/InstrProfilingFile.c +++ b/lib/profile/InstrProfilingFile.c @@ -1,9 +1,8 @@ /*===- InstrProfilingFile.c - Write instrumentation to a file -------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -112,6 +111,15 @@ static uint32_t fileWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, return 0; } +/* TODO: make buffer size controllable by an internal option, and compiler can pass the size + to runtime via a variable. */ +static uint32_t orderFileWriter(FILE *File, const uint32_t *DataStart) { + if (fwrite(DataStart, sizeof(uint32_t), INSTR_ORDER_FILE_BUFFER_SIZE, File) != + INSTR_ORDER_FILE_BUFFER_SIZE) + return 1; + return 0; +} + static void initFileWriter(ProfDataWriter *This, FILE *File) { This->Write = fileWriter; This->WriterCtx = File; @@ -261,6 +269,27 @@ static int writeFile(const char *OutputName) { return RetVal; } +/* Write order data to file \c OutputName. */ +static int writeOrderFile(const char *OutputName) { + int RetVal; + FILE *OutputFile; + + OutputFile = fopen(OutputName, "w"); + + if (!OutputFile) { + PROF_WARN("can't open file with mode ab: %s\n", OutputName); + return -1; + } + + FreeHook = &free; + setupIOBuffer(); + const uint32_t *DataBegin = __llvm_profile_begin_orderfile(); + RetVal = orderFileWriter(OutputFile, DataBegin); + + fclose(OutputFile); + return RetVal; +} + static void truncateCurrentFile(void) { const char *Filename; char *FilenameBuf; @@ -649,6 +678,62 @@ int __llvm_profile_dump(void) { return rc; } +/* Order file data will be saved in a file with suffx .order. */ +static const char *OrderFileSuffix = ".order"; + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_write_file(void) { + int rc, Length, LengthBeforeAppend, SuffixLength; + const char *Filename; + char *FilenameBuf; + int PDeathSig = 0; + + SuffixLength = strlen(OrderFileSuffix); + Length = getCurFilenameLength() + SuffixLength; + FilenameBuf = (char *)COMPILER_RT_ALLOCA(Length + 1); + Filename = getCurFilename(FilenameBuf, 1); + + /* Check the filename. */ + if (!Filename) { + PROF_ERR("Failed to write file : %s\n", "Filename not set"); + return -1; + } + + /* Append order file suffix */ + LengthBeforeAppend = strlen(Filename); + memcpy(FilenameBuf + LengthBeforeAppend, OrderFileSuffix, SuffixLength); + FilenameBuf[LengthBeforeAppend + SuffixLength] = '\0'; + + /* Check if there is llvm/runtime version mismatch. */ + if (GET_VERSION(__llvm_profile_get_version()) != INSTR_PROF_RAW_VERSION) { + PROF_ERR("Runtime and instrumentation version mismatch : " + "expected %d, but get %d\n", + INSTR_PROF_RAW_VERSION, + (int)GET_VERSION(__llvm_profile_get_version())); + return -1; + } + + // Temporarily suspend getting SIGKILL when the parent exits. + PDeathSig = lprofSuspendSigKill(); + + /* Write order data to the file. */ + rc = writeOrderFile(Filename); + if (rc) + PROF_ERR("Failed to write file \"%s\": %s\n", Filename, strerror(errno)); + + // Restore SIGKILL. + if (PDeathSig == 1) + lprofRestoreSigKill(); + + return rc; +} + +COMPILER_RT_VISIBILITY +int __llvm_orderfile_dump(void) { + int rc = __llvm_orderfile_write_file(); + return rc; +} + static void writeFileWithoutReturn(void) { __llvm_profile_write_file(); } COMPILER_RT_VISIBILITY diff --git a/lib/profile/InstrProfilingInternal.h b/lib/profile/InstrProfilingInternal.h index 40540ab5f..66f8a0677 100644 --- a/lib/profile/InstrProfilingInternal.h +++ b/lib/profile/InstrProfilingInternal.h @@ -1,9 +1,8 @@ /*===- InstrProfiling.h- Support library for PGO instrumentation ----------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingMerge.c b/lib/profile/InstrProfilingMerge.c index 2397250fb..44dce7cc9 100644 --- a/lib/profile/InstrProfilingMerge.c +++ b/lib/profile/InstrProfilingMerge.c @@ -1,9 +1,8 @@ /*===- InstrProfilingMerge.c - Profile in-process Merging ---------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------===* |* This file defines the API needed for in-process merging of profile data diff --git a/lib/profile/InstrProfilingMergeFile.c b/lib/profile/InstrProfilingMergeFile.c index dc1bc9762..b853f15b4 100644 --- a/lib/profile/InstrProfilingMergeFile.c +++ b/lib/profile/InstrProfilingMergeFile.c @@ -1,9 +1,8 @@ /*===- InstrProfilingMergeFile.c - Profile in-process Merging ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* |*===----------------------------------------------------------------------=== |* This file defines APIs needed to support in-process merging for profile data @@ -21,6 +20,7 @@ /* Merge value profile data pointed to by SrcValueProfData into * in-memory profile counters pointed by to DstData. */ +COMPILER_RT_VISIBILITY void lprofMergeValueProfData(ValueProfData *SrcValueProfData, __llvm_profile_data *DstData) { unsigned I, S, V, DstIndex = 0; diff --git a/lib/profile/InstrProfilingNameVar.c b/lib/profile/InstrProfilingNameVar.c index 264568fbc..2d67a55b9 100644 --- a/lib/profile/InstrProfilingNameVar.c +++ b/lib/profile/InstrProfilingNameVar.c @@ -1,9 +1,8 @@ /*===- InstrProfilingNameVar.c - profile name variable setup -------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingPlatformDarwin.c b/lib/profile/InstrProfilingPlatformDarwin.c index 8931abadd..23bdb7f37 100644 --- a/lib/profile/InstrProfilingPlatformDarwin.c +++ b/lib/profile/InstrProfilingPlatformDarwin.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformDarwin.c - Profile data on Darwin ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -13,28 +12,31 @@ /* Use linker magic to find the bounds of the Data section. */ COMPILER_RT_VISIBILITY extern __llvm_profile_data - DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); + DataStart __asm("section$start$__DATA$" INSTR_PROF_DATA_SECT_NAME); COMPILER_RT_VISIBILITY extern __llvm_profile_data - DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME_STR); + DataEnd __asm("section$end$__DATA$" INSTR_PROF_DATA_SECT_NAME); COMPILER_RT_VISIBILITY extern char - NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); + NamesStart __asm("section$start$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY -extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME_STR); +extern char NamesEnd __asm("section$end$__DATA$" INSTR_PROF_NAME_SECT_NAME); COMPILER_RT_VISIBILITY extern uint64_t - CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); + CountersStart __asm("section$start$__DATA$" INSTR_PROF_CNTS_SECT_NAME); COMPILER_RT_VISIBILITY extern uint64_t - CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME_STR); + CountersEnd __asm("section$end$__DATA$" INSTR_PROF_CNTS_SECT_NAME); +COMPILER_RT_VISIBILITY +extern uint32_t + OrderFileStart __asm("section$start$__DATA$" INSTR_PROF_ORDERFILE_SECT_NAME); COMPILER_RT_VISIBILITY extern ValueProfNode - VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME_STR); + VNodesStart __asm("section$start$__DATA$" INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY extern ValueProfNode - VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME_STR); + VNodesEnd __asm("section$end$__DATA$" INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY const __llvm_profile_data *__llvm_profile_begin_data(void) { @@ -50,6 +52,8 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart; } COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformFuchsia.c b/lib/profile/InstrProfilingPlatformFuchsia.c index a50602ded..80beb4145 100644 --- a/lib/profile/InstrProfilingPlatformFuchsia.c +++ b/lib/profile/InstrProfilingPlatformFuchsia.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformFuchsia.c - Profile data Fuchsia platform ----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ /* @@ -71,7 +70,7 @@ static uint32_t lprofVMOWriter(ProfDataWriter *This, ProfDataIOVec *IOVecs, return -1; /* Create VMO to hold the profile data. */ - Status = _zx_vmo_create(0, 0, &__llvm_profile_vmo); + Status = _zx_vmo_create(0, ZX_VMO_RESIZABLE, &__llvm_profile_vmo); if (Status != ZX_OK) return -1; diff --git a/lib/profile/InstrProfilingPlatformLinux.c b/lib/profile/InstrProfilingPlatformLinux.c index 3764df1d8..becfe1fd9 100644 --- a/lib/profile/InstrProfilingPlatformLinux.c +++ b/lib/profile/InstrProfilingPlatformLinux.c @@ -1,9 +1,8 @@ /*===- InstrProfilingPlatformLinux.c - Profile data Linux platform ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -14,14 +13,15 @@ #include "InstrProfiling.h" -#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_SECT_NAME) -#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_SECT_NAME) -#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_SECT_NAME) -#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_SECT_NAME) -#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_SECT_NAME) -#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_SECT_NAME) -#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_SECT_NAME) -#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_SECT_NAME) +#define PROF_DATA_START INSTR_PROF_SECT_START(INSTR_PROF_DATA_COMMON) +#define PROF_DATA_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_DATA_COMMON) +#define PROF_NAME_START INSTR_PROF_SECT_START(INSTR_PROF_NAME_COMMON) +#define PROF_NAME_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_NAME_COMMON) +#define PROF_CNTS_START INSTR_PROF_SECT_START(INSTR_PROF_CNTS_COMMON) +#define PROF_CNTS_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_CNTS_COMMON) +#define PROF_ORDERFILE_START INSTR_PROF_SECT_START(INSTR_PROF_ORDERFILE_COMMON) +#define PROF_VNODES_START INSTR_PROF_SECT_START(INSTR_PROF_VNODES_COMMON) +#define PROF_VNODES_STOP INSTR_PROF_SECT_STOP(INSTR_PROF_VNODES_COMMON) /* Declare section start and stop symbols for various sections * generated by compiler instrumentation. @@ -30,6 +30,7 @@ extern __llvm_profile_data PROF_DATA_START COMPILER_RT_VISIBILITY; extern __llvm_profile_data PROF_DATA_STOP COMPILER_RT_VISIBILITY; extern uint64_t PROF_CNTS_START COMPILER_RT_VISIBILITY; extern uint64_t PROF_CNTS_STOP COMPILER_RT_VISIBILITY; +extern uint32_t PROF_ORDERFILE_START COMPILER_RT_VISIBILITY; extern char PROF_NAME_START COMPILER_RT_VISIBILITY; extern char PROF_NAME_STOP COMPILER_RT_VISIBILITY; extern ValueProfNode PROF_VNODES_START COMPILER_RT_VISIBILITY; @@ -37,11 +38,13 @@ extern ValueProfNode PROF_VNODES_STOP COMPILER_RT_VISIBILITY; /* Add dummy data to ensure the section is always created. */ __llvm_profile_data - __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME_STR); + __prof_data_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_DATA_SECT_NAME); uint64_t - __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME_STR); -char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME_STR); -ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME_STR); + __prof_cnts_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_CNTS_SECT_NAME); +uint32_t + __prof_orderfile_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_ORDERFILE_SECT_NAME); +char __prof_nms_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_NAME_SECT_NAME); +ValueProfNode __prof_vnodes_sect_data[0] COMPILER_RT_SECTION(INSTR_PROF_VNODES_SECT_NAME); COMPILER_RT_VISIBILITY const __llvm_profile_data * __llvm_profile_begin_data(void) { @@ -63,6 +66,9 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return &PROF_CNTS_STOP; } +COMPILER_RT_VISIBILITY uint32_t *__llvm_profile_begin_orderfile(void) { + return &PROF_ORDERFILE_START; +} COMPILER_RT_VISIBILITY ValueProfNode * __llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformOther.c b/lib/profile/InstrProfilingPlatformOther.c index 7c2f14cfc..56c5d8378 100644 --- a/lib/profile/InstrProfilingPlatformOther.c +++ b/lib/profile/InstrProfilingPlatformOther.c @@ -1,16 +1,17 @@ /*===- InstrProfilingPlatformOther.c - Profile data default platform ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ -#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ - !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) +#if !defined(__APPLE__) && !defined(__linux__) && !defined(__FreeBSD__) && \ + !(defined(__sun__) && defined(__svr4__)) && !defined(__NetBSD__) && \ + !defined(_WIN32) #include <stdlib.h> +#include <stdio.h> #include "InstrProfiling.h" @@ -20,6 +21,7 @@ static const char *NamesFirst = NULL; static const char *NamesLast = NULL; static uint64_t *CountersFirst = NULL; static uint64_t *CountersLast = NULL; +static uint32_t *OrderFileFirst = NULL; static const void *getMinAddr(const void *A1, const void *A2) { return A1 < A2 ? A1 : A2; @@ -81,6 +83,9 @@ COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_begin_counters(void) { return CountersFirst; } COMPILER_RT_VISIBILITY uint64_t *__llvm_profile_end_counters(void) { return CountersLast; } +/* TODO: correctly set up OrderFileFirst. */ +COMPILER_RT_VISIBILITY +uint32_t *__llvm_profile_begin_orderfile(void) { return OrderFileFirst; } COMPILER_RT_VISIBILITY ValueProfNode *__llvm_profile_begin_vnodes(void) { diff --git a/lib/profile/InstrProfilingPlatformWindows.c b/lib/profile/InstrProfilingPlatformWindows.c new file mode 100644 index 000000000..81b708bb2 --- /dev/null +++ b/lib/profile/InstrProfilingPlatformWindows.c @@ -0,0 +1,68 @@ +/*===- InstrProfilingPlatformWindows.c - Profile data on Windows ----------===*\ +|* +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +|* +\*===----------------------------------------------------------------------===*/ + +#include "InstrProfiling.h" + +#if defined(_WIN32) + +#if defined(_MSC_VER) +/* Merge read-write sections into .data. */ +#pragma comment(linker, "/MERGE:.lprfc=.data") +#pragma comment(linker, "/MERGE:.lprfd=.data") +#pragma comment(linker, "/MERGE:.lprfv=.data") +#pragma comment(linker, "/MERGE:.lprfnd=.data") +/* Do *NOT* merge .lprfn and .lcovmap into .rdata. llvm-cov must be able to find + * after the fact. + */ + +/* Allocate read-only section bounds. */ +#pragma section(".lprfn$A", read) +#pragma section(".lprfn$Z", read) + +/* Allocate read-write section bounds. */ +#pragma section(".lprfd$A", read, write) +#pragma section(".lprfd$Z", read, write) +#pragma section(".lprfc$A", read, write) +#pragma section(".lprfc$Z", read, write) +#pragma section(".lorderfile$A", read, write) +#pragma section(".lprfnd$A", read, write) +#pragma section(".lprfnd$Z", read, write) +#endif + +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$A") DataStart = {0}; +__llvm_profile_data COMPILER_RT_SECTION(".lprfd$Z") DataEnd = {0}; + +const char COMPILER_RT_SECTION(".lprfn$A") NamesStart = '\0'; +const char COMPILER_RT_SECTION(".lprfn$Z") NamesEnd = '\0'; + +uint64_t COMPILER_RT_SECTION(".lprfc$A") CountersStart; +uint64_t COMPILER_RT_SECTION(".lprfc$Z") CountersEnd; +uint32_t COMPILER_RT_SECTION(".lorderfile$A") OrderFileStart; + +ValueProfNode COMPILER_RT_SECTION(".lprfnd$A") VNodesStart; +ValueProfNode COMPILER_RT_SECTION(".lprfnd$Z") VNodesEnd; + +const __llvm_profile_data *__llvm_profile_begin_data(void) { + return &DataStart + 1; +} +const __llvm_profile_data *__llvm_profile_end_data(void) { return &DataEnd; } + +const char *__llvm_profile_begin_names(void) { return &NamesStart + 1; } +const char *__llvm_profile_end_names(void) { return &NamesEnd; } + +uint64_t *__llvm_profile_begin_counters(void) { return &CountersStart + 1; } +uint64_t *__llvm_profile_end_counters(void) { return &CountersEnd; } +uint32_t *__llvm_profile_begin_orderfile(void) { return &OrderFileStart; } + +ValueProfNode *__llvm_profile_begin_vnodes(void) { return &VNodesStart + 1; } +ValueProfNode *__llvm_profile_end_vnodes(void) { return &VNodesEnd; } + +ValueProfNode *CurrentVNode = &VNodesStart + 1; +ValueProfNode *EndVNode = &VNodesEnd; + +#endif diff --git a/lib/profile/InstrProfilingPort.h b/lib/profile/InstrProfilingPort.h index ede6aaaf7..da5b5c0f8 100644 --- a/lib/profile/InstrProfilingPort.h +++ b/lib/profile/InstrProfilingPort.h @@ -1,9 +1,8 @@ /*===- InstrProfilingPort.h- Support library for PGO instrumentation ------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingRuntime.cc b/lib/profile/InstrProfilingRuntime.cc index eb8307498..679186ef8 100644 --- a/lib/profile/InstrProfilingRuntime.cc +++ b/lib/profile/InstrProfilingRuntime.cc @@ -1,9 +1,8 @@ //===- InstrProfilingRuntime.cpp - PGO runtime initialization -------------===// // -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception // //===----------------------------------------------------------------------===// diff --git a/lib/profile/InstrProfilingUtil.c b/lib/profile/InstrProfilingUtil.c index 083bf14a3..5e479ae71 100644 --- a/lib/profile/InstrProfilingUtil.c +++ b/lib/profile/InstrProfilingUtil.c @@ -1,9 +1,8 @@ /*===- InstrProfilingUtil.c - Support library for PGO instrumentation -----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -81,7 +80,7 @@ void *lprofPtrFetchAdd(void **Mem, long ByteIncr) { #endif -#ifdef _MSC_VER +#ifdef _WIN32 COMPILER_RT_VISIBILITY int lprofGetHostName(char *Name, int Len) { WCHAR Buffer[COMPILER_RT_MAX_HOSTLEN]; DWORD BufferSize = sizeof(Buffer); diff --git a/lib/profile/InstrProfilingUtil.h b/lib/profile/InstrProfilingUtil.h index 147677fc8..9cd0860fd 100644 --- a/lib/profile/InstrProfilingUtil.h +++ b/lib/profile/InstrProfilingUtil.h @@ -1,9 +1,8 @@ /*===- InstrProfilingUtil.h - Support library for PGO instrumentation -----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/InstrProfilingValue.c b/lib/profile/InstrProfilingValue.c index c7b01a570..b7c71768c 100644 --- a/lib/profile/InstrProfilingValue.c +++ b/lib/profile/InstrProfilingValue.c @@ -1,9 +1,8 @@ /*===- InstrProfilingValue.c - Support library for PGO instrumentation ----===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ @@ -32,7 +31,7 @@ static int hasNonDefaultValsPerSite = 0; * allocated by the compiler. */ COMPILER_RT_VISIBILITY ValueProfNode lprofValueProfNodes[INSTR_PROF_VNODE_POOL_SIZE] COMPILER_RT_SECTION( - COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME_STR); + COMPILER_RT_SEG INSTR_PROF_VNODES_SECT_NAME); #endif COMPILER_RT_VISIBILITY uint32_t VPMaxNumValsPerSite = diff --git a/lib/profile/InstrProfilingWriter.c b/lib/profile/InstrProfilingWriter.c index 7c6061d2c..d910cbb8f 100644 --- a/lib/profile/InstrProfilingWriter.c +++ b/lib/profile/InstrProfilingWriter.c @@ -1,9 +1,8 @@ /*===- InstrProfilingWriter.c - Write instrumentation to a file or buffer -===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ diff --git a/lib/profile/WindowsMMap.h b/lib/profile/WindowsMMap.h index 51a130b31..c8d6250f4 100644 --- a/lib/profile/WindowsMMap.h +++ b/lib/profile/WindowsMMap.h @@ -1,9 +1,8 @@ /*===- WindowsMMap.h - Support library for PGO instrumentation ------------===*\ |* -|* The LLVM Compiler Infrastructure -|* -|* This file is distributed under the University of Illinois Open Source -|* License. See LICENSE.TXT for details. +|* Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +|* See https://llvm.org/LICENSE.txt for license information. +|* SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception |* \*===----------------------------------------------------------------------===*/ |