goldenChat base source add

This commit is contained in:
aidev
2026-05-23 15:11:48 +09:00
commit a4ea7762b5
2081 changed files with 1155760 additions and 0 deletions
@@ -0,0 +1,145 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources;
import org.ta4j.core.BarSeries;
import java.io.InputStream;
import java.time.Duration;
import java.time.Instant;
/**
* Common interface for data sources that load BarSeries using business domain
* concepts (ticker, interval, date range).
* <p>
* This interface abstracts away implementation details (files, APIs, databases)
* and allows users to work with trading domain concepts. File-based
* implementations will search for files matching the ticker/interval/date range
* criteria, while API-based implementations will fetch data from remote
* sources.
* <p>
* <strong>Domain-Driven Usage:</strong>
*
* <pre>
* // All data sources work with business concepts
* BarSeriesDataSource yahoo = new YahooFinanceHttpBarSeriesDataSource(true);
* BarSeriesDataSource csv = new CsvFileBarSeriesDataSource();
* BarSeriesDataSource json = new JsonFileBarSeriesDataSource();
*
* // Same interface, different implementations
* BarSeries aapl = yahoo.loadSeries("AAPL", Duration.ofDays(1), Instant.parse("2023-01-01T00:00:00Z"),
* Instant.parse("2023-12-31T23:59:59Z"));
*
* BarSeries btc = csv.loadSeries("BTC-USD", Duration.ofDays(1), Instant.parse("2023-01-01T00:00:00Z"),
* Instant.parse("2023-12-31T23:59:59Z"));
* </pre>
* <p>
* <strong>Direct Source Loading:</strong>
* <p>
* For cases where you know the exact source identifier (filename, URL, etc.),
* use {@link #loadSeries(String)}:
*
* <pre>
* // Direct filename loading (bypasses search logic)
* BarSeries series = csv.loadSeries("AAPL-PT1D-20130102_20131231.csv");
* </pre>
* <p>
* <strong>Implementation Behavior:</strong>
* <ul>
* <li><strong>File-based sources</strong> (CsvFileBarSeriesDataSource,
* JsonFileBarSeriesDataSource, BitStampCsvTradesFileBarSeriesDataSource):
* Search for files matching ticker/interval/date range patterns in the
* classpath or configured directories. The exact filename pattern is
* implementation-specific.</li>
* <li><strong>API-based sources</strong> (YahooFinanceHttpBarSeriesDataSource):
* Fetch data from the API. If caching is enabled, will first check for cached
* files matching the criteria.</li>
* </ul>
*
* @since 0.20
*/
public interface BarSeriesDataSource {
/**
* Loads a BarSeries using business domain concepts.
* <p>
* This is the primary method for loading data. Implementations interpret the
* parameters according to their capabilities:
* <ul>
* <li><strong>File-based sources</strong>: Search for files matching the
* ticker, interval, and date range. The search pattern is
* implementation-specific (e.g.,
* "{ticker}_bars_from_{startDate}_{endDate}.csv").</li>
* <li><strong>API-based sources</strong>: Fetch data from the API for the
* specified ticker, interval, and date range. May check cache first if caching
* is enabled.</li>
* </ul>
*
* @param ticker the ticker symbol or identifier (e.g., "AAPL", "BTC-USD",
* "MSFT")
* @param interval the bar interval (e.g., Duration.ofDays(1) for daily bars,
* Duration.ofHours(1) for hourly bars)
* @param start the start date/time for the data range (inclusive)
* @param end the end date/time for the data range (inclusive)
* @return a BarSeries containing the loaded data, or null if no matching data
* is found or loading fails
* @throws IllegalArgumentException if any parameter is invalid (e.g., null
* ticker, negative interval, start after end)
*/
BarSeries loadSeries(String ticker, Duration interval, Instant start, Instant end);
/**
* Loads a BarSeries directly from a known source identifier.
* <p>
* This method bypasses the search/fetch logic and loads directly from the
* specified source. Use this when you know the exact source identifier (e.g.,
* filename, URL, resource name).
* <p>
* For file-based sources, this is typically a filename or resource path. For
* API-based sources, this might be a cached file path or a direct API endpoint.
*
* @param source the source identifier (filename, resource name, URL, etc.)
* @return a BarSeries containing the loaded data, or null if loading fails
* @throws IllegalArgumentException if the source parameter is invalid or
* unsupported
*/
BarSeries loadSeries(String source);
/**
* Loads a BarSeries from an InputStream.
* <p>
* This method is optional - implementations that don't support InputStream
* loading should throw {@link UnsupportedOperationException}.
* <p>
* The caller is responsible for closing the InputStream after this method
* returns. Implementations should not close the stream unless they fully
* consume it.
*
* @param inputStream the input stream containing the data
* @return a BarSeries containing the loaded data, or null if loading fails
* @throws UnsupportedOperationException if this data source doesn't support
* InputStream loading
* @throws IllegalArgumentException if the inputStream is null or invalid
*/
default BarSeries loadSeries(InputStream inputStream) {
throw new UnsupportedOperationException(
"InputStream loading not supported by " + this.getClass().getSimpleName());
}
/**
* Returns the source name for this data source. This is used for building file
* search patterns, cache file names, and other source-specific identifiers.
* <p>
* For example, a Yahoo Finance data source would return "YahooFinance", which
* would be used to build cache file names like "YahooFinance-AAPL-1d-...".
* <p>
* File-based sources that don't use a source prefix (e.g., generic CSV files)
* should return an empty string.
*
* @return the source name, or an empty string if no source prefix is used
*/
default String getSourceName() {
return "";
}
}
@@ -0,0 +1,315 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.BarSeries;
import org.ta4j.core.BaseBarSeriesBuilder;
import org.ta4j.core.num.Num;
import ta4jexamples.datasources.file.AbstractFileBarSeriesDataSource;
import java.io.InputStream;
import java.io.InputStreamReader;
import java.time.Duration;
import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.ListIterator;
/**
* This class builds a Ta4j bar series from a Bitstamp CSV file containing
* trades. It reads trade-level data (timestamp, price, volume) and aggregates
* them into OHLCV bars suitable for technical analysis.
* <p>
* Implements {@link BarSeriesDataSource} to support domain-driven loading by
* ticker, interval, and date range. Searches for Bitstamp CSV files matching
* the specified criteria in the classpath.
*/
public class BitStampCsvTradesFileBarSeriesDataSource extends AbstractFileBarSeriesDataSource {
private static final Logger LOG = LogManager.getLogger(BitStampCsvTradesFileBarSeriesDataSource.class);
private static final String DEFAULT_BITSTAMP_FILE = "Bitstamp-BTC-USD-PT5M-20131125_20131201.csv";
/**
* Creates a new BitStampCsvTradesFileBarSeriesDataSource with "Bitstamp" as the
* source name.
*/
public BitStampCsvTradesFileBarSeriesDataSource() {
super("Bitstamp");
}
@Override
protected String getFileExtension() {
return "csv";
}
@Override
protected BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
Instant start, Instant end) {
// Try exact pattern with interval-appropriate format:
// {sourceName}-{ticker}-{interval}-{startDateTime}_{endDateTime}.csv
String exactPattern = sourcePrefix + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_"
+ endDateTimeStr + ".csv";
BarSeries series = loadBitstampSeries(exactPattern);
if (series != null && !series.isEmpty()) {
return filterAndAggregateSeries(series, interval, start, end);
}
// Fallback to date-only format for backward compatibility with existing files
String exactPatternDateOnly = sourcePrefix + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_"
+ endDateStr + ".csv";
series = loadBitstampSeries(exactPatternDateOnly);
if (series != null && !series.isEmpty()) {
return filterAndAggregateSeries(series, interval, start, end);
}
// Try broader pattern: {sourceName}-{ticker}-*-{startDateTime}_*.csv
String broaderPattern = sourcePrefix + ticker.toUpperCase() + "-*-" + startDateTimeStr + "_*.csv";
series = searchAndLoadBitstampFile(broaderPattern, interval, start, end);
if (series != null && !series.isEmpty()) {
return series;
}
// Fallback to date-only format for broader pattern
String broaderPatternDateOnly = sourcePrefix + ticker.toUpperCase() + "-*-" + startDateStr + "_*.csv";
series = searchAndLoadBitstampFile(broaderPatternDateOnly, interval, start, end);
if (series != null && !series.isEmpty()) {
return series;
}
// Try even broader: {sourceName}-{ticker}-*.csv (then filter by date range)
String broadestPattern = sourcePrefix + ticker.toUpperCase() + "-*.csv";
series = searchAndLoadBitstampFile(broadestPattern, interval, start, end);
if (series != null && !series.isEmpty()) {
return series;
}
return null;
}
@Override
public BarSeries loadSeries(String source) {
if (source == null || source.trim().isEmpty()) {
throw new IllegalArgumentException("Source cannot be null or empty");
}
return loadBitstampSeries(source);
}
/**
* Searches for a Bitstamp CSV file matching the pattern and loads it if found.
*
* @param pattern the filename pattern to search for (supports wildcards)
* @param interval the desired bar interval
* @param start the start date
* @param end the end date
* @return the loaded and filtered BarSeries, or null if not found
*/
private BarSeries searchAndLoadBitstampFile(String pattern, Duration interval, Instant start, Instant end) {
// Try direct pattern match as resource
if (!pattern.contains("*")) {
BarSeries series = loadBitstampSeries(pattern);
if (series != null && !series.isEmpty()) {
return filterAndAggregateSeries(series, interval, start, end);
}
}
// For wildcard patterns, try common variations
String[] variations = { pattern.replace("*", "PT5M"), pattern.replace("*", "PT1D") };
for (String variation : variations) {
BarSeries series = loadBitstampSeries(variation);
if (series != null && !series.isEmpty()) {
return filterAndAggregateSeries(series, interval, start, end);
}
}
return null;
}
/**
* Filters a series to the date range and re-aggregates with the specified
* interval.
*
* @param series the series to filter and aggregate
* @param interval the desired bar interval
* @param start the start date (inclusive)
* @param end the end date (inclusive)
* @return a new BarSeries with bars within the date range and specified
* interval, or null if no bars match
*/
private BarSeries filterAndAggregateSeries(BarSeries series, Duration interval, Instant start, Instant end) {
if (series == null || series.isEmpty()) {
return null;
}
// Filter trades within date range and re-aggregate with new interval
// This is a simplified implementation - in practice, you'd want to
// re-aggregate from the original trade data
var filteredSeries = new BaseBarSeriesBuilder().withName(series.getName()).build();
int barsInDateRange = 0;
int barsWithMatchingInterval = 0;
Duration actualInterval = null;
for (int i = 0; i < series.getBarCount(); i++) {
var bar = series.getBar(i);
Instant barEnd = bar.getEndTime();
if (!barEnd.isBefore(start) && !barEnd.isAfter(end)) {
barsInDateRange++;
if (actualInterval == null) {
actualInterval = bar.getTimePeriod();
}
// If interval matches, add as-is; otherwise would need re-aggregation
if (bar.getTimePeriod().equals(interval)) {
barsWithMatchingInterval++;
filteredSeries.addBar(bar);
}
}
}
// Log warning if bars exist in date range but intervals don't match
if (barsInDateRange > 0 && barsWithMatchingInterval == 0 && actualInterval != null) {
LOG.warn(
"Found {} bars within date range [{} to {}], but bar interval ({}) does not match requested interval ({}). "
+ "Re-aggregation from original trade data is required but not implemented. Returning null.",
barsInDateRange, start, end, actualInterval, interval);
}
return filteredSeries.isEmpty() ? null : filteredSeries;
}
/**
* Loads a bar series from the default Bitstamp CSV file. The method reads trade
* data from a CSV file containing Bitstamp exchange trades and converts it into
* a bar series format suitable for technical analysis.
*
* @return the bar series from Bitstamp (bitcoin exchange) trades
*/
public static BarSeries loadBitstampSeries() {
return loadBitstampSeries(DEFAULT_BITSTAMP_FILE);
}
/**
* Loads a bar series from a specified Bitstamp CSV file. The method reads trade
* data from a CSV file containing Bitstamp exchange trades and converts it into
* a bar series format suitable for technical analysis.
*
* @param bitstampCsvFile the path to the CSV file containing Bitstamp trade
* data
* @return the bar series built from the Bitstamp trades data
*/
public static BarSeries loadBitstampSeries(String bitstampCsvFile) {
// Reading all lines of the CSV file
InputStream stream = BitStampCsvTradesFileBarSeriesDataSource.class.getClassLoader()
.getResourceAsStream(bitstampCsvFile);
List<String[]> lines = null;
if (stream == null) {
LOG.debug("CSV file not found in classpath: {}", bitstampCsvFile);
return null;
}
try (final var csvReader = new com.opencsv.CSVReader(new InputStreamReader(stream))) {
lines = csvReader.readAll();
lines.remove(0); // Removing header line
} catch (Exception ioe) {
LOG.error("Unable to load trades from CSV", ioe);
}
var series = new BaseBarSeriesBuilder().withName(bitstampCsvFile).build();
if ((lines != null) && !lines.isEmpty()) {
// Getting the first and last trades timestamps
Instant beginTime = null;
Instant endTime = null;
try {
beginTime = Instant.ofEpochMilli(Long.parseLong(lines.get(0)[0]) * 1000);
endTime = Instant.ofEpochMilli(Long.parseLong(lines.get(lines.size() - 1)[0]) * 1000);
} catch (NumberFormatException nfe) {
LOG.error("Invalid trade timestamp format in CSV: {}", nfe.getMessage());
return null;
}
if (beginTime.isAfter(endTime)) {
beginTime = endTime;
// Since the CSV file has the most recent trades at the top of the file, we'll
// reverse the list to feed
// the List<Bar> correctly.
Collections.reverse(lines);
}
// build the list of populated bars (default 5-minute bars)
buildSeries(series, beginTime, endTime, 300, lines);
}
return series.isEmpty() ? null : series;
}
/**
* Builds a list of populated bars from csv data.
*
* @param beginTime the begin time of the whole period
* @param endTime the end time of the whole period
* @param duration the bar duration (in seconds)
* @param lines the csv data returned by CSVReader.readAll()
*/
private static void buildSeries(BarSeries series, Instant beginTime, Instant endTime, int duration,
List<String[]> lines) {
Duration barDuration = Duration.ofSeconds(duration);
Instant barEndTime = beginTime;
ListIterator<String[]> iterator = lines.listIterator();
// line number of trade data
do {
// build a bar
barEndTime = barEndTime.plus(barDuration);
var bar = series.barBuilder().timePeriod(barDuration).endTime(barEndTime).volume(0).amount(0).build();
do {
// get a trade
String[] tradeLine = iterator.next();
Instant tradeTimeStamp;
try {
tradeTimeStamp = Instant.ofEpochMilli(Long.parseLong(tradeLine[0]) * 1000);
} catch (NumberFormatException nfe) {
LOG.warn("Invalid trade timestamp format in CSV line, skipping trade: {}",
tradeLine.length > 0 ? tradeLine[0] : "empty line", nfe);
continue;
}
// if the trade happened during the bar
if (bar.inPeriod(tradeTimeStamp)) {
// add the trade to the bar
Num tradePrice;
Num tradeVolume;
try {
tradePrice = series.numFactory().numOf(Double.parseDouble(tradeLine[1]));
tradeVolume = series.numFactory().numOf(Double.parseDouble(tradeLine[2]));
} catch (NumberFormatException nfe) {
LOG.warn(
"Invalid trade price or volume format in CSV line, skipping trade: price={}, volume={}",
tradeLine.length > 1 ? tradeLine[1] : "missing",
tradeLine.length > 2 ? tradeLine[2] : "missing", nfe);
continue;
}
bar.addTrade(tradeVolume, tradePrice);
} else {
// the trade happened after the end of the bar
// go to the next bar but stay with the same trade (don't increment i)
// this break will drop us after the inner "while", skipping the increment
break;
}
} while (iterator.hasNext());
// if the bar has any trades add it to the bars list
// this is where the break drops to
if (bar.getTrades() > 0) {
series.addBar(bar);
}
} while (barEndTime.isBefore(endTime));
}
public static void main(String[] args) {
BarSeries series = BitStampCsvTradesFileBarSeriesDataSource.loadBitstampSeries();
LOG.debug("Series: {} ({})", series.getName(), series.getSeriesPeriodDescription());
LOG.debug("Number of bars: {}", series.getBarCount());
LOG.debug("First bar: \n\tVolume: {}\n\tNumber of trades: {}\n\tClose price: {}", series.getBar(0).getVolume(),
series.getBar(0).getTrades(), series.getBar(0).getClosePrice());
}
}
@@ -0,0 +1,867 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.Bar;
import org.ta4j.core.BarSeries;
import org.ta4j.core.BaseBarSeriesBuilder;
import ta4jexamples.datasources.http.AbstractHttpBarSeriesDataSource;
import ta4jexamples.datasources.http.DefaultHttpClientWrapper;
import ta4jexamples.datasources.http.HttpClientWrapper;
import ta4jexamples.datasources.http.HttpResponseWrapper;
import ta4jexamples.datasources.json.AdaptiveBarSeriesTypeAdapter;
import java.io.IOException;
import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeMap;
/**
* Loads OHLCV data from Coinbase Advanced Trade API.
* <p>
* This loader fetches historical price data from Coinbase's public market data
* API without requiring authentication. It supports all Coinbase trading pairs
* (e.g., BTC-USD, ETH-USD).
* <p>
* <strong>Example usage:</strong>
*
* <pre>
* // Load 1 year of daily data for Bitcoin (using days)
* BarSeries series = CoinbaseHttpBarSeriesDataSource.loadSeries("BTC-USD", 365);
*
* // Load 500 bars of hourly data for Ethereum (using bar count)
* BarSeries ethSeries = CoinbaseHttpBarSeriesDataSource.loadSeries("ETH-USD", CoinbaseInterval.ONE_HOUR, 500);
*
* // Load data for a specific date range
* Instant start = Instant.parse("2023-01-01T00:00:00Z");
* Instant end = Instant.parse("2023-12-31T23:59:59Z");
* BarSeries btcSeries = CoinbaseHttpBarSeriesDataSource.loadSeries("BTC-USD", CoinbaseInterval.ONE_DAY, start, end);
* </pre>
* <p>
* <strong>Response Caching:</strong> To enable response caching for faster
* subsequent requests, use the constructor with {@code enableResponseCaching}:
*
* <pre>
* CoinbaseHttpBarSeriesDataSource loader = new CoinbaseHttpBarSeriesDataSource(true);
* BarSeries series = loader.loadSeriesInstance("BTC-USD", CoinbaseInterval.ONE_DAY, start, end);
* </pre>
* <p>
* To use a custom cache directory, use the constructor with
* {@code responseCacheDir}:
*
* <pre>
* CoinbaseHttpBarSeriesDataSource loader = new CoinbaseHttpBarSeriesDataSource("/path/to/cache");
* BarSeries series = loader.loadSeriesInstance("BTC-USD", CoinbaseInterval.ONE_DAY, start, end);
* </pre>
* <p>
* When caching is enabled, responses are saved to the cache directory (default:
* {@code temp/responses}) and reused for requests within the cache validity
* period (based on the interval). For example, daily data is cached for the
* day, 15-minute data is cached for 15 minutes, etc. Historical data (end date
* in the past) is cached indefinitely.
* <p>
* <strong>Unit Testing:</strong> For unit testing with a mock HttpClient, use
* the constructor:
*
* <pre>
* HttpClientWrapper mockHttpClient = mock(HttpClientWrapper.class);
* CoinbaseHttpBarSeriesDataSource loader = new CoinbaseHttpBarSeriesDataSource(mockHttpClient);
* // Use loader instance methods or inject into your code
* </pre>
* <p>
* <strong>API Limits:</strong> Coinbase API has a maximum of 350 candles per
* request. This implementation automatically paginates large requests into
* multiple API calls and merges the results.
* <p>
* <strong>Note:</strong> This uses Coinbase's public market data endpoint which
* does not require authentication. For production use with higher rate limits,
* consider using authenticated endpoints.
*
* @since 0.20
*/
public class CoinbaseHttpBarSeriesDataSource extends AbstractHttpBarSeriesDataSource {
public static final String COINBASE_API_URL = "https://api.coinbase.com/api/v3/brokerage/market/products/";
public static final int MAX_CANDLES_PER_REQUEST = 350;
private static final Logger LOG = LogManager.getLogger(CoinbaseHttpBarSeriesDataSource.class);
@Override
public String getSourceName() {
return "Coinbase";
}
private static final HttpClientWrapper DEFAULT_HTTP_CLIENT = new DefaultHttpClientWrapper();
private static final CoinbaseHttpBarSeriesDataSource DEFAULT_INSTANCE = new CoinbaseHttpBarSeriesDataSource(
DEFAULT_HTTP_CLIENT);
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with a default HttpClient. For
* unit testing, use {@link #CoinbaseHttpBarSeriesDataSource(HttpClientWrapper)}
* to inject a mock HttpClientWrapper.
*/
public CoinbaseHttpBarSeriesDataSource() {
super(DEFAULT_HTTP_CLIENT, false);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with a default HttpClient and
* caching option.
*
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
*/
public CoinbaseHttpBarSeriesDataSource(boolean enableResponseCaching) {
super(DEFAULT_HTTP_CLIENT, enableResponseCaching);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with a default HttpClient and
* custom cache directory. Response caching is automatically enabled when a
* cache directory is specified.
*
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
*/
public CoinbaseHttpBarSeriesDataSource(String responseCacheDir) {
super(DEFAULT_HTTP_CLIENT, responseCacheDir);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified
* HttpClientWrapper. This constructor allows dependency injection of a mock
* HttpClientWrapper for unit testing.
*
* @param httpClient the HttpClientWrapper to use for API requests (can be a
* mock for testing)
*/
public CoinbaseHttpBarSeriesDataSource(HttpClientWrapper httpClient) {
super(httpClient, false);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified
* HttpClientWrapper and caching option. This constructor allows dependency
* injection of a mock HttpClientWrapper for unit testing and enables response
* caching.
*
* @param httpClient the HttpClientWrapper to use for API requests
* (can be a mock for testing)
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
*/
public CoinbaseHttpBarSeriesDataSource(HttpClientWrapper httpClient, boolean enableResponseCaching) {
super(httpClient, enableResponseCaching);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified HttpClient.
* This is a convenience constructor that wraps the HttpClient in a
* DefaultHttpClientWrapper.
*
* @param httpClient the HttpClient to use for API requests
*/
public CoinbaseHttpBarSeriesDataSource(HttpClient httpClient) {
super(httpClient, false);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified HttpClient
* and caching option.
*
* @param httpClient the HttpClient to use for API requests
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
*/
public CoinbaseHttpBarSeriesDataSource(HttpClient httpClient, boolean enableResponseCaching) {
super(httpClient, enableResponseCaching);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified
* HttpClientWrapper and custom cache directory. Response caching is
* automatically enabled when a cache directory is specified.
*
* @param httpClient the HttpClientWrapper to use for API requests (can be
* a mock for testing)
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
*/
public CoinbaseHttpBarSeriesDataSource(HttpClientWrapper httpClient, String responseCacheDir) {
super(httpClient, responseCacheDir);
}
/**
* Creates a new CoinbaseHttpBarSeriesDataSource with the specified HttpClient
* and custom cache directory. Response caching is automatically enabled when a
* cache directory is specified.
*
* @param httpClient the HttpClient to use for API requests
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
*/
public CoinbaseHttpBarSeriesDataSource(HttpClient httpClient, String responseCacheDir) {
super(httpClient, responseCacheDir);
}
/**
* Loads historical OHLCV data for a given product ID within a specified date
* range. This is the base method that all other convenience methods delegate
* to.
* <p>
* <strong>Automatic Pagination:</strong> If the requested date range would
* exceed 350 candles (Coinbase's maximum per request), this method
* automatically splits the request into multiple smaller chunks, fetches them
* sequentially, and merges the results into a single BarSeries. This ensures
* reliable data retrieval for large date ranges while respecting API limits.
*
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
* @param interval the bar interval (must be one of the supported Coinbase
* intervals)
* @param startDateTime the start date/time for the data range (inclusive)
* @param endDateTime the end date/time for the data range (inclusive)
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String productId, CoinbaseInterval interval, Instant startDateTime,
Instant endDateTime) {
return DEFAULT_INSTANCE.loadSeriesInstance(productId, interval, startDateTime, endDateTime);
}
/**
* Loads historical OHLCV data for a given product ID with a specified number of
* bars. The end date/time is set to the current time, and the start date/time
* is calculated based on the bar count and interval.
* <p>
* <strong>Note:</strong> If the calculated date range would exceed 350 candles,
* this method will automatically paginate the request into multiple API calls
* and merge the results. This ensures reliable data retrieval for large bar
* counts.
*
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
* @param interval the bar interval (must be one of the supported Coinbase
* intervals)
* @param barCount the number of bars to fetch
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String productId, CoinbaseInterval interval, int barCount) {
if (barCount <= 0) {
LOG.error("Bar count must be greater than 0");
return null;
}
Instant endDateTime = Instant.now();
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
Instant startDateTime = endDateTime.minus(totalDuration);
return loadSeries(productId, interval, startDateTime, endDateTime);
}
/**
* Loads historical OHLCV data for a given product ID with daily bars.
* Convenience method that uses the number of days to calculate the date range.
*
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
* @param days the number of days of historical data to fetch
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String productId, int days) {
return loadSeries(productId, CoinbaseInterval.ONE_DAY, days);
}
/**
* Loads historical OHLCV data for a given product ID with a specified interval.
* Convenience method that uses the number of days to calculate the date range.
*
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
* @param days the number of days of historical data to fetch
* @param interval the bar interval (must be one of the supported Coinbase
* intervals)
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String productId, int days, CoinbaseInterval interval) {
if (days <= 0) {
LOG.error("Days must be greater than 0");
return null;
}
Instant endDateTime = Instant.now();
Instant startDateTime = endDateTime.minusSeconds(days * 86400L);
return loadSeries(productId, interval, startDateTime, endDateTime);
}
/**
* Parses the Coinbase API JSON response into a BarSeries using
* AdaptiveBarSeriesTypeAdapter.
* <p>
* This method reuses the existing AdaptiveBarSeriesTypeAdapter which already
* supports Coinbase format and handles null values. The adapter correctly
* interprets Coinbase's "start" field as the start of the candle period and
* calculates the end time as start + interval. The known interval from the API
* request is used directly.
*
* @param jsonResponse the JSON response string from Coinbase API
* @param productId the product ID (used as series name)
* @param barInterval the known bar interval from the API request
* @return a BarSeries containing the parsed data, or null if parsing fails
*/
private static BarSeries parseCoinbaseResponse(String jsonResponse, String productId, Duration barInterval) {
try {
JsonObject root = JsonParser.parseString(jsonResponse).getAsJsonObject();
// Use AdaptiveBarSeriesTypeAdapter's static helper method which handles
// Coinbase format correctly (treats "start" as start time, calculates end as
// start + interval)
// and uses the known interval from the API request
BarSeries series = AdaptiveBarSeriesTypeAdapter.parseCoinbaseFormat(root, productId, barInterval);
if (series == null || series.isEmpty()) {
LOG.error("No candles found in Coinbase response for product: {}", productId);
return null;
}
LOG.debug("Successfully loaded {} bars for product {}", series.getBarCount(), productId);
return series;
} catch (Exception e) {
LOG.error("Error parsing Coinbase response for product {}: {}", productId, e.getMessage(), e);
return null;
}
}
/**
* Merges multiple BarSeries into a single BarSeries, removing duplicates and
* sorting chronologically. Uses a TreeMap keyed by timestamp to automatically
* handle deduplication and sorting.
*
* @param chunks list of BarSeries to merge
* @param productId the product ID (for the merged series name)
* @param barInterval the bar interval
* @return a merged BarSeries
*/
private static BarSeries mergeBarSeries(List<BarSeries> chunks, String productId, Duration barInterval) {
// Use TreeMap to automatically sort by timestamp and deduplicate
TreeMap<Instant, BarData> barMap = new TreeMap<>();
// Collect all bars from all chunks
for (BarSeries chunk : chunks) {
for (int i = 0; i < chunk.getBarCount(); i++) {
var bar = chunk.getBar(i);
Instant endTime = bar.getEndTime();
// If we already have a bar at this timestamp, keep the first one
barMap.putIfAbsent(endTime, new BarData(bar));
}
}
// Build the merged series
BarSeries merged = new BaseBarSeriesBuilder().withName(productId).build();
for (BarData barData : barMap.values()) {
merged.barBuilder()
.timePeriod(barInterval)
.endTime(barData.endTime)
.openPrice(barData.open)
.highPrice(barData.high)
.lowPrice(barData.low)
.closePrice(barData.close)
.volume(barData.volume)
.amount(0)
.add();
}
LOG.debug("Merged {} chunks into {} unique bars for product {}", chunks.size(), merged.getBarCount(),
productId);
return merged;
}
/**
* Instance method that loads historical OHLCV data for a given product ID with
* a specified number of bars. The end date/time is set to the current time, and
* the start date/time is calculated based on the bar count and interval.
*
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
* @param interval the bar interval (must be one of the supported Coinbase
* intervals)
* @param barCount the number of bars to fetch
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, int barCount) {
return loadSeriesInstance(productId, interval, barCount, null);
}
/**
* Instance method that loads historical OHLCV data for a given product ID with
* a specified number of bars and optional notes for cache file naming. The end
* date/time is set to the current time, and the start date/time is calculated
* based on the bar count and interval.
*
* @param productId the product ID (e.g., "BTC-USD", "ETH-USD")
* @param interval the bar interval (must be one of the supported Coinbase
* intervals)
* @param barCount the number of bars to fetch
* @param notes optional notes to include in cache filename (for uniqueness,
* e.g., test identifiers)
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, int barCount, String notes) {
if (barCount <= 0) {
LOG.error("Bar count must be greater than 0");
return null;
}
Instant endDateTime = Instant.now();
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
Instant startDateTime = endDateTime.minus(totalDuration);
return loadSeriesInstance(productId, interval, startDateTime, endDateTime, notes);
}
@Override
public BarSeries loadSeries(String productId, Duration interval, Instant start, Instant end) {
if (productId == null || productId.trim().isEmpty()) {
throw new IllegalArgumentException("Product ID cannot be null or empty");
}
if (interval == null || interval.isNegative() || interval.isZero()) {
throw new IllegalArgumentException("Interval must be positive");
}
if (start == null || end == null) {
throw new IllegalArgumentException("Start and end dates cannot be null");
}
if (start.isAfter(end)) {
throw new IllegalArgumentException("Start date must be before or equal to end date");
}
// Map Duration to CoinbaseInterval
CoinbaseInterval cbInterval = mapDurationToInterval(interval);
if (cbInterval == null) {
LOG.warn("Unsupported interval duration: {}. Falling back to ONE_DAY", interval);
cbInterval = CoinbaseInterval.ONE_DAY;
}
return loadSeriesInstance(productId, cbInterval, start, end);
}
@Override
public BarSeries loadSeries(String source) {
if (source == null || source.trim().isEmpty()) {
throw new IllegalArgumentException("Source cannot be null or empty");
}
// Check if it's a cache file path
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
if (source.startsWith(responseCacheDir) || (!sourcePrefix.isEmpty() && source.contains(sourcePrefix))) {
Path cacheFile = Paths.get(source);
if (Files.exists(cacheFile)) {
String cachedResponse = readFromCache(cacheFile);
if (cachedResponse != null) {
// Try to extract product ID from filename
String filename = cacheFile.getFileName().toString();
// Format: {sourceName}-PRODUCTID-INTERVAL-START-END[_NOTES].json
// Remove extension
String baseName = filename.replace(".json", "");
String[] parts = baseName.split("-");
if (parts.length >= 5) {
String productId = parts[1];
// Try to determine interval from filename
CoinbaseInterval interval = CoinbaseInterval.ONE_DAY; // Default
try {
interval = parseIntervalFromApiValue(parts[2]);
} catch (IllegalArgumentException e) {
LOG.debug("Could not parse interval from filename, using default: {}", e.getMessage());
}
return parseCoinbaseResponse(cachedResponse, productId, interval.getDuration());
}
}
}
}
// If not a cache file, return null (could be extended to parse other formats)
return null;
}
/**
* Maps a Duration to the closest matching CoinbaseInterval.
*
* @param duration the duration to map
* @return the matching CoinbaseInterval, or null if no close match is found
*/
private CoinbaseInterval mapDurationToInterval(Duration duration) {
long seconds = duration.getSeconds();
for (CoinbaseInterval interval : CoinbaseInterval.values()) {
if (interval.getDuration().getSeconds() == seconds) {
return interval;
}
}
return null;
}
/**
* Parses a CoinbaseInterval from its API value string.
*
* @param apiValue the API value (e.g., "ONE_MINUTE", "ONE_DAY")
* @return the matching CoinbaseInterval
* @throws IllegalArgumentException if no matching interval is found
*/
private CoinbaseInterval parseIntervalFromApiValue(String apiValue) {
for (CoinbaseInterval interval : CoinbaseInterval.values()) {
if (interval.getApiValue().equals(apiValue)) {
return interval;
}
}
throw new IllegalArgumentException("Unknown interval API value: " + apiValue);
}
/**
* Instance method that performs the actual loading logic. This method uses the
* instance's HttpClient (which can be injected for testing).
*/
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, Instant startDateTime,
Instant endDateTime) {
return loadSeriesInstance(productId, interval, startDateTime, endDateTime, null);
}
/**
* Instance method that performs the actual loading logic with optional notes.
* This method uses the instance's HttpClient (which can be injected for
* testing).
*
* @param productId the product ID
* @param interval the interval
* @param startDateTime the start date/time
* @param endDateTime the end date/time
* @param notes optional notes to include in cache filename (for
* uniqueness)
* @return the BarSeries or null if request fails
*/
public BarSeries loadSeriesInstance(String productId, CoinbaseInterval interval, Instant startDateTime,
Instant endDateTime, String notes) {
if (productId == null || productId.trim().isEmpty()) {
LOG.error("Product ID cannot be null or empty");
return null;
}
if (startDateTime == null || endDateTime == null) {
LOG.error("Start and end date/time cannot be null");
return null;
}
if (startDateTime.isAfter(endDateTime)) {
LOG.error("Start date/time must be before or equal to end date/time");
return null;
}
// Calculate if we need pagination (max 350 candles per request)
Duration requestedRange = Duration.between(startDateTime, endDateTime);
long requestedBars = requestedRange.dividedBy(interval.getDuration());
if (requestedBars > MAX_CANDLES_PER_REQUEST) {
LOG.debug(
"Requested date range would result in {} bars (max {}). "
+ "Splitting into multiple requests and combining results.",
requestedBars, MAX_CANDLES_PER_REQUEST);
return loadSeriesPaginated(productId, interval, startDateTime, endDateTime, notes);
}
// Single request for smaller ranges
return loadSeriesSingleRequest(productId, interval, startDateTime, endDateTime, notes);
}
/**
* Generates the cache file path for a given request.
*
* @param productId the product ID
* @param interval the interval
* @param startDateTime the start date/time (will be truncated)
* @param endDateTime the end date/time (will be truncated)
* @param notes optional notes section to append to filename (can be
* null or empty)
* @return the cache file path
*/
private Path getCacheFilePath(String productId, CoinbaseInterval interval, Instant startDateTime,
Instant endDateTime, String notes) {
return getCacheFilePath(productId, startDateTime, endDateTime, interval.getDuration(), notes);
}
/**
* Generates the cache file path for a given request (without notes).
*
* @param productId the product ID
* @param interval the interval
* @param startDateTime the start date/time (will be truncated)
* @param endDateTime the end date/time (will be truncated)
* @return the cache file path
*/
private Path getCacheFilePath(String productId, CoinbaseInterval interval, Instant startDateTime,
Instant endDateTime) {
return getCacheFilePath(productId, interval, startDateTime, endDateTime, null);
}
/**
* Makes a single API request for the specified date range with optional notes.
* This is used for requests that don't exceed 350 candles. If caching is
* enabled, checks cache first before making the API request.
*
* @param productId the product ID
* @param interval the interval
* @param startDateTime the start date/time
* @param endDateTime the end date/time
* @param notes optional notes to include in cache filename (for
* uniqueness)
* @return the BarSeries or null if request fails
*/
private BarSeries loadSeriesSingleRequest(String productId, CoinbaseInterval interval, Instant startDateTime,
Instant endDateTime, String notes) {
// Check cache first if caching is enabled
if (enableResponseCaching) {
// Try exact match first (with or without notes)
Path cacheFile = getCacheFilePath(productId, interval, startDateTime, endDateTime, notes);
if (isCacheValid(cacheFile, interval.getDuration(), endDateTime)) {
String cachedResponse = readFromCache(cacheFile);
if (cachedResponse != null) {
LOG.debug("Using cached response for {} ({} to {})", productId, startDateTime, endDateTime);
return parseCoinbaseResponse(cachedResponse, productId, interval.getDuration());
}
}
// Also try without notes (for backward compatibility)
if (notes != null && !notes.trim().isEmpty()) {
Path cacheFileNoNotes = getCacheFilePath(productId, interval, startDateTime, endDateTime);
if (isCacheValid(cacheFileNoNotes, interval.getDuration(), endDateTime)) {
String cachedResponse = readFromCache(cacheFileNoNotes);
if (cachedResponse != null) {
LOG.debug("Using cached response for {} ({} to {})", productId, startDateTime, endDateTime);
return parseCoinbaseResponse(cachedResponse, productId, interval.getDuration());
}
}
}
}
try {
String encodedProductId = URLEncoder.encode(productId.trim(), StandardCharsets.UTF_8);
long startTimestamp = startDateTime.getEpochSecond();
long endTimestamp = endDateTime.getEpochSecond();
String url = String.format("%s%s/candles?start=%d&end=%d&granularity=%s&limit=%d", COINBASE_API_URL,
encodedProductId, startTimestamp, endTimestamp, interval.getApiValue(), MAX_CANDLES_PER_REQUEST);
LOG.trace("Fetching data from Coinbase: {}", url);
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.header("Accept", "application/json")
.timeout(Duration.ofSeconds(30))
.GET()
.build();
HttpResponseWrapper<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() != 200) {
LOG.error("Coinbase API returned status code: {}", response.statusCode());
return null;
}
String responseBody = response.body();
LOG.trace("Response body: {}", responseBody);
// Cache the response if caching is enabled
if (enableResponseCaching) {
Path cacheFile = getCacheFilePath(productId, interval, startDateTime, endDateTime, notes);
writeToCache(cacheFile, responseBody);
}
return parseCoinbaseResponse(responseBody, productId, interval.getDuration());
} catch (IOException | InterruptedException e) {
LOG.error("Error fetching data from Coinbase for product {}: {}", productId, e.getMessage(), e);
return null;
}
}
/**
* Loads data by splitting a large date range into multiple smaller requests
* (pagination). Each chunk respects the 350 candle limit, and results are
* merged chronologically.
*
* @param productId the product ID
* @param interval the bar interval
* @param startDateTime the start date/time
* @param endDateTime the end date/time
* @param notes optional notes to include in cache filename (for
* uniqueness)
* @return a BarSeries containing all merged data, or null if all requests fail
*/
private BarSeries loadSeriesPaginated(String productId, CoinbaseInterval interval, Instant startDateTime,
Instant endDateTime, String notes) {
List<BarSeries> chunks = new ArrayList<>();
Instant currentStart = startDateTime;
int requestCount = 0;
// Calculate chunk size (350 candles worth of time)
Duration chunkSize = interval.getDuration().multipliedBy(MAX_CANDLES_PER_REQUEST);
// Calculate number of chunks needed
Duration totalRange = Duration.between(startDateTime, endDateTime);
int estimatedChunks = (int) Math.ceil((double) totalRange.toSeconds() / chunkSize.toSeconds());
LOG.trace("Splitting request into approximately {} chunks", estimatedChunks);
while (currentStart.isBefore(endDateTime)) {
// Calculate chunk end time (don't exceed the requested end time)
Instant chunkEnd = currentStart.plus(chunkSize);
if (chunkEnd.isAfter(endDateTime)) {
chunkEnd = endDateTime;
}
requestCount++;
LOG.trace("Fetching chunk {}/? ({} to {})", requestCount, currentStart, chunkEnd);
BarSeries chunk = loadSeriesSingleRequest(productId, interval, currentStart, chunkEnd, notes);
if (chunk != null && chunk.getBarCount() > 0) {
chunks.add(chunk);
LOG.trace("Successfully loaded chunk {} with {} bars", requestCount, chunk.getBarCount());
} else {
LOG.warn("Chunk {} returned no data or failed", requestCount);
}
// Move to next chunk (start from the end of current chunk)
currentStart = chunkEnd;
// If we've reached the end, break
if (chunkEnd.equals(endDateTime) || !currentStart.isBefore(endDateTime)) {
break;
}
// Add a small delay between requests to avoid rate limiting
try {
Thread.sleep(100); // 100ms delay between requests
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.warn("Interrupted during pagination delay");
break;
}
}
if (chunks.isEmpty()) {
LOG.error("All paginated requests failed for product {}", productId);
return null;
}
LOG.debug("Successfully fetched {} chunks, merging {} total bars", chunks.size(),
chunks.stream().mapToInt(BarSeries::getBarCount).sum());
return mergeBarSeries(chunks, productId, interval.getDuration());
}
/**
* Supported intervals for Coinbase API. These correspond to the intervals that
* Coinbase's Advanced Trade API supports.
*/
public enum CoinbaseInterval {
/**
* 1 minute bars
*/
ONE_MINUTE(Duration.ofMinutes(1), "ONE_MINUTE"),
/**
* 5 minute bars
*/
FIVE_MINUTE(Duration.ofMinutes(5), "FIVE_MINUTE"),
/**
* 15 minute bars
*/
FIFTEEN_MINUTE(Duration.ofMinutes(15), "FIFTEEN_MINUTE"),
/**
* 30 minute bars
*/
THIRTY_MINUTE(Duration.ofMinutes(30), "THIRTY_MINUTE"),
/**
* 1 hour bars
*/
ONE_HOUR(Duration.ofHours(1), "ONE_HOUR"),
/**
* 2 hour bars
*/
TWO_HOUR(Duration.ofHours(2), "TWO_HOUR"),
/**
* 4 hour bars
*/
FOUR_HOUR(Duration.ofHours(4), "FOUR_HOUR"),
/**
* 6 hour bars
*/
SIX_HOUR(Duration.ofHours(6), "SIX_HOUR"),
/**
* 1 day bars
*/
ONE_DAY(Duration.ofDays(1), "ONE_DAY");
private final Duration duration;
private final String apiValue;
CoinbaseInterval(Duration duration, String apiValue) {
this.duration = duration;
this.apiValue = apiValue;
}
/**
* Returns the Duration for this interval.
*
* @return the Duration
*/
public Duration getDuration() {
return duration;
}
/**
* Returns the API string value for this interval.
*
* @return the API string value
*/
public String getApiValue() {
return apiValue;
}
}
/**
* Helper class to hold bar data during merging.
*/
private static class BarData {
final Instant endTime;
final double open;
final double high;
final double low;
final double close;
final double volume;
BarData(Bar bar) {
this.endTime = bar.getEndTime();
this.open = bar.getOpenPrice().doubleValue();
this.high = bar.getHighPrice().doubleValue();
this.low = bar.getLowPrice().doubleValue();
this.close = bar.getClosePrice().doubleValue();
this.volume = bar.getVolume().doubleValue();
}
}
}
@@ -0,0 +1,250 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources;
import com.opencsv.CSVParserBuilder;
import com.opencsv.CSVReader;
import com.opencsv.CSVReaderBuilder;
import com.opencsv.exceptions.CsvValidationException;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.BarSeries;
import org.ta4j.core.BaseBarSeriesBuilder;
import ta4jexamples.datasources.file.AbstractFileBarSeriesDataSource;
import java.io.IOException;
import java.io.InputStreamReader;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.time.LocalDate;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* This class builds a Ta4j bar series from a CSV file containing bars.
* <p>
* Implements {@link BarSeriesDataSource} to support domain-driven loading by
* ticker, interval, and date range. Searches for CSV files matching the
* specified criteria in the classpath.
*/
public class CsvFileBarSeriesDataSource extends AbstractFileBarSeriesDataSource {
private static final Logger LOG = LogManager.getLogger(CsvFileBarSeriesDataSource.class);
private static final DateTimeFormatter DATE_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd");
private static final String DEFAULT_APPLE_BAR_FILE = "AAPL-PT1D-20130102_20131231.csv";
/**
* Creates a new CsvFileBarSeriesDataSource with no source prefix.
*/
public CsvFileBarSeriesDataSource() {
super("");
}
@Override
protected String getFileExtension() {
return "csv";
}
@Override
protected BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
Instant start, Instant end) {
// Try exact pattern with interval-appropriate format:
// {ticker}-{interval}-{startDateTime}_{endDateTime}.csv
String exactPattern = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_" + endDateTimeStr
+ ".csv";
BarSeries series = loadCsvSeries(exactPattern);
if (series != null && !series.isEmpty()) {
return series;
}
// Fallback to date-only format for existing files
String exactPatternDateOnly = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_" + endDateStr
+ ".csv";
series = loadCsvSeries(exactPatternDateOnly);
if (series != null && !series.isEmpty()) {
return series;
}
// Try broader pattern: {ticker}-*-{startDateTime}_*.csv
String broaderPattern = ticker.toUpperCase() + "-*-" + startDateTimeStr + "_*.csv";
series = searchAndLoadCsvFile(broaderPattern, start, end);
if (series != null && !series.isEmpty()) {
return series;
}
// Fallback to date-only format for broader pattern
String broaderPatternDateOnly = ticker.toUpperCase() + "-*-" + startDateStr + "_*.csv";
series = searchAndLoadCsvFile(broaderPatternDateOnly, start, end);
if (series != null && !series.isEmpty()) {
return series;
}
// Try even broader: {ticker}-*.csv (then filter by date range)
String broadestPattern = ticker.toUpperCase() + "-*.csv";
series = searchAndLoadCsvFile(broadestPattern, start, end);
if (series != null && !series.isEmpty()) {
return filterSeriesByDateRange(series, start, end);
}
return null;
}
@Override
public BarSeries loadSeries(String source) {
if (source == null || source.trim().isEmpty()) {
throw new IllegalArgumentException("Source cannot be null or empty");
}
return loadCsvSeries(source);
}
/**
* Searches for a CSV file matching the pattern and loads it if found.
*
* @param pattern the filename pattern to search for (supports wildcards)
* @param start the start date (for validation)
* @param end the end date (for validation)
* @return the loaded BarSeries, or null if not found
*/
private BarSeries searchAndLoadCsvFile(String pattern, Instant start, Instant end) {
// Try direct pattern match as resource
if (!pattern.contains("*")) {
return loadCsvSeries(pattern);
}
// Count wildcards to determine replacement strategy
long wildcardCount = pattern.chars().filter(ch -> ch == '*').count();
if (wildcardCount == 1) {
// Single wildcard: try common interval values
String[] intervalVariations = { "PT1D", "PT5M", "PT1H", "" };
for (String interval : intervalVariations) {
String variation = pattern.replaceFirst("\\*", interval);
BarSeries series = loadCsvSeries(variation);
if (series != null && !series.isEmpty()) {
return series;
}
}
} else if (wildcardCount >= 2) {
// Multiple wildcards: replace the first (interval) with common values,
// and the second (end date) with the actual end date from parameters
String[] intervalVariations = { "PT1D", "PT5M", "PT1H" };
// Format end date in both date-only and datetime formats
String endDateStr = end.atZone(ZoneOffset.UTC).format(FILENAME_DATE_FORMAT);
String endDateTimeStr = end.atZone(ZoneOffset.UTC)
.format(getDateTimeFormatterForInterval(Duration.ofDays(1)));
for (String interval : intervalVariations) {
// Replace first wildcard with interval, second with end date (date-only format)
String variation = pattern.replaceFirst("\\*", interval).replaceFirst("\\*", endDateStr);
BarSeries series = loadCsvSeries(variation);
if (series != null && !series.isEmpty()) {
return series;
}
// Also try with datetime format for the end date
variation = pattern.replaceFirst("\\*", interval).replaceFirst("\\*", endDateTimeStr);
series = loadCsvSeries(variation);
if (series != null && !series.isEmpty()) {
return series;
}
}
}
return null;
}
/**
* Loads a bar series from the default CSV file.
*
* @return the bar series loaded from the default CSV file
*/
public static BarSeries loadSeriesFromFile() {
return loadSeriesFromFile(DEFAULT_APPLE_BAR_FILE);
}
/**
* Loads a bar series from the specified CSV file. This is a convenience method
* that delegates to {@link #loadCsvSeries(String)}.
*
* @param csvFile the path to the CSV file containing bar data
* @return the bar series loaded from the specified CSV file
*/
public static BarSeries loadSeriesFromFile(String csvFile) {
return loadCsvSeries(csvFile);
}
/**
* Loads a bar series from a CSV file with the specified filename. The CSV file
* is expected to contain stock market data with the following columns: date,
* open price, high price, low price, close price, and volume. The date format
* is expected to match the predefined DATE_FORMAT.
*
* @param filename the name of the CSV file to load
* @return the bar series containing stock data loaded from the specified CSV
* file, or null if the file is not found or empty
*/
public static BarSeries loadCsvSeries(String filename) {
var stream = CsvFileBarSeriesDataSource.class.getClassLoader().getResourceAsStream(filename);
if (stream == null) {
LOG.debug("CSV file not found in classpath: {}", filename);
return null;
}
var series = new BaseBarSeriesBuilder().withName(filename).build();
try (stream) {
try (InputStreamReader reader = new InputStreamReader(stream, StandardCharsets.UTF_8)) {
try (CSVReader csvReader = new CSVReaderBuilder(reader)
.withCSVParser(new CSVParserBuilder().withSeparator(',').build())
.withSkipLines(1)
.build()) {
String[] line;
while ((line = csvReader.readNext()) != null) {
Instant date = LocalDate.parse(line[0], DATE_FORMAT).atStartOfDay(ZoneOffset.UTC).toInstant();
double open = Double.parseDouble(line[1]);
double high = Double.parseDouble(line[2]);
double low = Double.parseDouble(line[3]);
double close = Double.parseDouble(line[4]);
double volume = Double.parseDouble(line[5]);
series.barBuilder()
.timePeriod(Duration.ofDays(1))
.endTime(date)
.openPrice(open)
.closePrice(close)
.highPrice(high)
.lowPrice(low)
.volume(volume)
.amount(0)
.add();
}
} catch (CsvValidationException e) {
LOG.error("Unable to load bars from CSV. File is not valid csv.", e);
}
}
} catch (IOException ioe) {
LOG.error("Unable to load bars from CSV", ioe);
} catch (NumberFormatException nfe) {
LOG.error("Error while parsing value", nfe);
return null;
}
return series.isEmpty() ? null : series;
}
public static void main(String[] args) {
BarSeries series = CsvFileBarSeriesDataSource.loadSeriesFromFile();
LOG.debug("Series: {} ({})", series.getName(), series.getSeriesPeriodDescription());
LOG.debug("Number of bars: {}", series.getBarCount());
if (series.isEmpty()) {
LOG.warn("Series is empty - no bars loaded from CSV file. Skipping first bar details.");
} else {
LOG.debug("First bar: \n\tVolume: {}\n\tOpen price: {}\n\tClose price: {}", series.getBar(0).getVolume(),
series.getBar(0).getOpenPrice(), series.getBar(0).getClosePrice());
}
}
}
@@ -0,0 +1,177 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import com.google.gson.stream.JsonReader;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.BarSeries;
import ta4jexamples.datasources.file.AbstractFileBarSeriesDataSource;
import ta4jexamples.datasources.json.AdaptiveBarSeriesTypeAdapter;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
/**
* A data source for BarSeries objects that can adapt to different JSON formats.
* This class provides methods to load BarSeries data from JSON format,
* specifically supporting multiple exchange formats such as Binance and
* Coinbase. It uses Gson with a custom TypeAdapter to handle the
* deserialization process. The data source can read from either an InputStream
* or a file path.
* <p>
* Implements {@link BarSeriesDataSource} to support domain-driven loading by
* ticker, interval, and date range. Searches for JSON files matching the
* specified criteria in the classpath.
*
* @since 0.19
*/
public class JsonFileBarSeriesDataSource extends AbstractFileBarSeriesDataSource {
private static final Gson TYPEADAPTER_GSON = new GsonBuilder()
.registerTypeAdapter(BarSeries.class, new AdaptiveBarSeriesTypeAdapter())
.create();
private static final Logger LOG = LogManager.getLogger(JsonFileBarSeriesDataSource.class);
/**
* Creates a new JsonFileBarSeriesDataSource with no source prefix.
*/
public JsonFileBarSeriesDataSource() {
super("");
}
/**
* Default instance for backward compatibility with static method calls. Use
* this instance when migrating from static methods to instance methods.
*/
public static final JsonFileBarSeriesDataSource DEFAULT_INSTANCE = new JsonFileBarSeriesDataSource();
@Override
protected String getFileExtension() {
return "json";
}
@Override
protected BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
Instant start, Instant end) {
// Try with exchange prefixes (Coinbase-, Binance-)
// NOTE: All branches must call filterSeriesByDateRange() to ensure data is
// filtered
// to the requested date range, even when files contain broader date ranges.
String[] exchangePrefixes = { "Coinbase-", "Binance-" };
for (String exchange : exchangePrefixes) {
// Try exact pattern with interval-appropriate format:
// {Exchange}-{ticker}-{interval}-{startDateTime}_{endDateTime}.json
String pattern = exchange + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_"
+ endDateTimeStr + ".json";
BarSeries series = loadFromSource(pattern);
if (series != null && !series.isEmpty()) {
return filterSeriesByDateRange(series, start, end);
}
// Fallback to date-only format for existing files
// NOTE: Date-only format may match files with broader date ranges, so filtering
// is required
String patternDateOnly = exchange + ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_"
+ endDateStr + ".json";
series = loadFromSource(patternDateOnly);
if (series != null && !series.isEmpty()) {
return filterSeriesByDateRange(series, start, end);
}
}
// Try without exchange prefix as fallback (for generic JSON files)
String pattern = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateTimeStr + "_" + endDateTimeStr
+ ".json";
BarSeries series = loadFromSource(pattern);
if (series != null && !series.isEmpty()) {
return filterSeriesByDateRange(series, start, end);
}
// Fallback to date-only format
// NOTE: Date-only format may match files with broader date ranges, so filtering
// is required
String patternDateOnly = ticker.toUpperCase() + "-" + intervalStr + "-" + startDateStr + "_" + endDateStr
+ ".json";
series = loadFromSource(patternDateOnly);
if (series != null && !series.isEmpty()) {
return filterSeriesByDateRange(series, start, end);
}
return null;
}
@Override
public BarSeries loadSeries(String source) {
return loadFromSource(source);
}
@Override
public BarSeries loadSeries(InputStream inputStream) {
return loadFromStream(inputStream);
}
/**
* Internal implementation for loading from a file.
*/
private BarSeries loadFromSource(String source) {
if (source == null || source.trim().isEmpty()) {
return null;
}
try (FileInputStream fis = new FileInputStream(source)) {
return loadFromStream(fis);
} catch (Exception e) {
// Try as classpath resource
InputStream resourceStream = JsonFileBarSeriesDataSource.class.getClassLoader().getResourceAsStream(source);
if (resourceStream != null) {
try (resourceStream) {
return loadFromStream(resourceStream);
} catch (Exception resourceException) {
LOG.debug("Unable to load bars from classpath resource: {}", source, resourceException);
return null;
}
}
LOG.debug("Unable to load bars from file: {}", source, e);
return null;
}
}
/**
* Internal implementation for loading from InputStream.
* <p>
* This method fully consumes the stream but does not close it, as per the
* interface contract. The caller is responsible for closing the stream.
*/
private BarSeries loadFromStream(InputStream inputStream) {
if (inputStream == null) {
LOG.debug("Input stream is null, returning null");
return null;
}
// Read the stream fully into a String without closing it
// This ensures we fully consume the stream while respecting the contract
// that the caller is responsible for closing it
String jsonContent;
try {
jsonContent = new String(inputStream.readAllBytes(), StandardCharsets.UTF_8);
} catch (IOException e) {
LOG.debug("Unable to read from input stream", e);
return null;
}
// Parse the JSON content from the String
try (JsonReader reader = new JsonReader(new java.io.StringReader(jsonContent))) {
return TYPEADAPTER_GSON.fromJson(reader, BarSeries.class);
} catch (Exception e) {
LOG.debug("Unable to load bars from JSON using TypeAdapter", e);
return null;
}
}
}
@@ -0,0 +1,978 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources;
import com.google.gson.JsonArray;
import com.google.gson.JsonObject;
import com.google.gson.JsonParser;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.Bar;
import org.ta4j.core.BarSeries;
import org.ta4j.core.BaseBarSeriesBuilder;
import ta4jexamples.datasources.http.AbstractHttpBarSeriesDataSource;
import ta4jexamples.datasources.http.DefaultHttpClientWrapper;
import ta4jexamples.datasources.http.HttpClientWrapper;
import ta4jexamples.datasources.http.HttpResponseWrapper;
import java.io.IOException;
import java.net.URI;
import java.net.URLEncoder;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.List;
import java.util.TreeMap;
/**
* Loads OHLCV data from Yahoo Finance API.
* <p>
* This loader fetches historical price data from Yahoo Finance's public API
* without requiring an API key. It supports stocks, ETFs, and cryptocurrencies.
* <p>
* <strong>Example usage:</strong>
*
* <pre>
* // Load 1 year of daily data for Apple stock (using days)
* BarSeries series = YahooFinanceHttpBarSeriesDataSource.loadSeries("AAPL", 365);
*
* // Load 500 bars of hourly data for Bitcoin (using bar count)
* BarSeries btcSeries = YahooFinanceHttpBarSeriesDataSource.loadSeries("BTC-USD", YahooFinanceInterval.HOUR_1, 500);
*
* // Load data for a specific date range
* Instant start = Instant.parse("2023-01-01T00:00:00Z");
* Instant end = Instant.parse("2023-12-31T23:59:59Z");
* BarSeries msftSeries = YahooFinanceHttpBarSeriesDataSource.loadSeries("MSFT", YahooFinanceInterval.DAY_1, start,
* end);
* </pre>
* <p>
* <strong>Response Caching:</strong> To enable response caching for faster
* subsequent requests, use the constructor with {@code enableResponseCaching}:
*
* <pre>
* YahooFinanceHttpBarSeriesDataSource loader = new YahooFinanceHttpBarSeriesDataSource(true);
* BarSeries series = loader.loadSeriesInstance("AAPL", YahooFinanceInterval.DAY_1, start, end);
* </pre>
* <p>
* To use a custom cache directory, use the constructor with
* {@code responseCacheDir}:
*
* <pre>
* YahooFinanceHttpBarSeriesDataSource loader = new YahooFinanceHttpBarSeriesDataSource("/path/to/cache");
* BarSeries series = loader.loadSeriesInstance("AAPL", YahooFinanceInterval.DAY_1, start, end);
* </pre>
* <p>
* When caching is enabled, responses are saved to the cache directory (default:
* {@code temp/responses}) and reused for requests within the cache validity
* period (based on the interval). For example, daily data is cached for the
* day, 15-minute data is cached for 15 minutes, etc. Historical data (end date
* in the past) is cached indefinitely.
* <p>
* <strong>Unit Testing:</strong> For unit testing with a mock HttpClient, use
* the constructor:
*
* <pre>
* HttpClientWrapper mockHttpClient = mock(HttpClientWrapper.class);
* YahooFinanceHttpBarSeriesDataSource loader = new YahooFinanceHttpBarSeriesDataSource(mockHttpClient);
* // Use loader instance methods or inject into your code
* </pre>
* <p>
* <strong>Note:</strong> Yahoo Finance is an unofficial API and may have rate
* limits or availability issues. For production use, consider using official
* APIs like Alpha Vantage, Polygon.io, or IEX Cloud.
*
* @since 0.20
*/
public class YahooFinanceHttpBarSeriesDataSource extends AbstractHttpBarSeriesDataSource {
public static final String YAHOO_FINANCE_API_URL = "https://query1.finance.yahoo.com/v8/finance/chart/";
private static final Logger LOG = LogManager.getLogger(YahooFinanceHttpBarSeriesDataSource.class);
@Override
public String getSourceName() {
return "YahooFinance";
}
private static final HttpClientWrapper DEFAULT_HTTP_CLIENT = new DefaultHttpClientWrapper();
private static final YahooFinanceHttpBarSeriesDataSource DEFAULT_INSTANCE = new YahooFinanceHttpBarSeriesDataSource(
DEFAULT_HTTP_CLIENT);
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with a default HttpClient.
* For unit testing, use
* {@link #YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper)} to inject a
* mock HttpClientWrapper.
*/
public YahooFinanceHttpBarSeriesDataSource() {
super(DEFAULT_HTTP_CLIENT, false);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with a default HttpClient
* and caching option.
*
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
*/
public YahooFinanceHttpBarSeriesDataSource(boolean enableResponseCaching) {
super(DEFAULT_HTTP_CLIENT, enableResponseCaching);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with a default HttpClient
* and custom cache directory. Response caching is automatically enabled when a
* cache directory is specified.
*
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
*/
public YahooFinanceHttpBarSeriesDataSource(String responseCacheDir) {
super(DEFAULT_HTTP_CLIENT, responseCacheDir);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
* HttpClientWrapper. This constructor allows dependency injection of a mock
* HttpClientWrapper for unit testing.
*
* @param httpClient the HttpClientWrapper to use for API requests (can be a
* mock for testing)
*/
public YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper httpClient) {
super(httpClient, false);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
* HttpClientWrapper and caching option. This constructor allows dependency
* injection of a mock HttpClientWrapper for unit testing and enables response
* caching.
*
* @param httpClient the HttpClientWrapper to use for API requests
* (can be a mock for testing)
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
*/
public YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper httpClient, boolean enableResponseCaching) {
super(httpClient, enableResponseCaching);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
* HttpClient. This is a convenience constructor that wraps the HttpClient in a
* DefaultHttpClientWrapper.
*
* @param httpClient the HttpClient to use for API requests
*/
public YahooFinanceHttpBarSeriesDataSource(HttpClient httpClient) {
super(httpClient, false);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
* HttpClient and caching option.
*
* @param httpClient the HttpClient to use for API requests
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
*/
public YahooFinanceHttpBarSeriesDataSource(HttpClient httpClient, boolean enableResponseCaching) {
super(httpClient, enableResponseCaching);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
* HttpClientWrapper and custom cache directory. Response caching is
* automatically enabled when a cache directory is specified.
*
* @param httpClient the HttpClientWrapper to use for API requests (can be
* a mock for testing)
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
*/
public YahooFinanceHttpBarSeriesDataSource(HttpClientWrapper httpClient, String responseCacheDir) {
super(httpClient, responseCacheDir);
}
/**
* Creates a new YahooFinanceHttpBarSeriesDataSource with the specified
* HttpClient and custom cache directory. Response caching is automatically
* enabled when a cache directory is specified.
*
* @param httpClient the HttpClient to use for API requests
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
*/
public YahooFinanceHttpBarSeriesDataSource(HttpClient httpClient, String responseCacheDir) {
super(httpClient, responseCacheDir);
}
/**
* Loads historical OHLCV data for a given ticker symbol within a specified date
* range. This is the base method that all other convenience methods delegate
* to.
* <p>
* <strong>Automatic Pagination:</strong> If the requested date range exceeds
* conservative API limits, this method automatically splits the request into
* multiple smaller chunks, fetches them sequentially, and merges the results
* into a single BarSeries. This ensures reliable data retrieval for large date
* ranges while respecting API rate limits.
* <p>
* <strong>API Limits:</strong> Yahoo Finance's unofficial API has practical
* limits:
* <ul>
* <li>Rate limits: ~2000 requests/hour per IP (may result in temporary bans if
* exceeded)</li>
* <li>Data range limits (approximate, may vary):
* <ul>
* <li>Intraday (1m-4h): Typically 60-90 days maximum per request</li>
* <li>Daily (1d): Typically 2-5 years maximum per request</li>
* <li>Weekly/Monthly (1wk, 1mo): Can request many years per request</li>
* </ul>
* </li>
* </ul>
* <p>
* <strong>Conservative Limits (triggers pagination):</strong>
* <ul>
* <li>Intraday (1m-4h): 30 days per chunk</li>
* <li>Hourly (1h, 4h): 60 days per chunk</li>
* <li>Daily (1d): 1 year per chunk</li>
* <li>Weekly/Monthly (1wk, 1mo): 5 years per chunk</li>
* </ul>
*
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
* "ETH-USD")
* @param interval the bar interval (must be one of the supported Yahoo
* Finance intervals)
* @param startDateTime the start date/time for the data range (inclusive)
* @param endDateTime the end date/time for the data range (inclusive)
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String ticker, YahooFinanceInterval interval, Instant startDateTime,
Instant endDateTime) {
return DEFAULT_INSTANCE.loadSeriesInstance(ticker, interval, startDateTime, endDateTime);
}
/**
* Loads historical OHLCV data for a given ticker symbol with a specified number
* of bars. The end date/time is set to the current time, and the start
* date/time is calculated based on the bar count and interval.
* <p>
* <strong>Note:</strong> If the calculated date range exceeds conservative API
* limits, this method will automatically paginate the request into multiple API
* calls and merge the results. This ensures reliable data retrieval for large
* bar counts.
*
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
* "ETH-USD")
* @param interval the bar interval (must be one of the supported Yahoo Finance
* intervals)
* @param barCount the number of bars to fetch
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String ticker, YahooFinanceInterval interval, int barCount) {
if (barCount <= 0) {
LOG.error("Bar count must be greater than 0");
return null;
}
Instant endDateTime = Instant.now();
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
Instant startDateTime = endDateTime.minus(totalDuration);
return loadSeries(ticker, interval, startDateTime, endDateTime);
}
/**
* Loads historical OHLCV data for a given ticker symbol with daily bars.
* Convenience method that uses the number of days to calculate the date range.
*
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD", "ETH-USD")
* @param days the number of days of historical data to fetch
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String ticker, int days) {
return loadSeries(ticker, YahooFinanceInterval.DAY_1, days);
}
/**
* Loads historical OHLCV data for a given ticker symbol with a specified
* interval. Convenience method that uses the number of days to calculate the
* date range.
*
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
* "ETH-USD")
* @param days the number of days of historical data to fetch
* @param interval the bar interval (must be one of the supported Yahoo Finance
* intervals)
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public static BarSeries loadSeries(String ticker, int days, YahooFinanceInterval interval) {
if (days <= 0) {
LOG.error("Days must be greater than 0");
return null;
}
Instant endDateTime = Instant.now();
Instant startDateTime = endDateTime.minusSeconds(days * 86400L);
return loadSeries(ticker, interval, startDateTime, endDateTime);
}
/**
* Parses the Yahoo Finance API JSON response into a BarSeries.
*/
private static BarSeries parseYahooFinanceResponse(String jsonResponse, String ticker, Duration barInterval) {
try {
JsonObject root = JsonParser.parseString(jsonResponse).getAsJsonObject();
JsonObject chart = root.getAsJsonObject("chart");
JsonArray results = chart.getAsJsonArray("result");
if (results == null || results.isEmpty()) {
LOG.error("No results found in Yahoo Finance response for ticker: {}", ticker);
return null;
}
JsonObject result = results.get(0).getAsJsonObject();
// Get timestamps array
JsonArray timestamps = result.getAsJsonArray("timestamp");
if (timestamps == null) {
LOG.error("No timestamp data found in Yahoo Finance response for ticker: {}", ticker);
return null;
}
JsonObject indicators = result.getAsJsonObject("indicators");
if (indicators == null) {
LOG.error("No indicators found in Yahoo Finance response for ticker: {}", ticker);
return null;
}
JsonArray quotes = indicators.getAsJsonArray("quote");
if (quotes == null || quotes.isEmpty()) {
LOG.error("No quote data found in Yahoo Finance response for ticker: {}", ticker);
return null;
}
JsonObject quote = quotes.get(0).getAsJsonObject();
JsonArray opens = quote.getAsJsonArray("open");
JsonArray highs = quote.getAsJsonArray("high");
JsonArray lows = quote.getAsJsonArray("low");
JsonArray closes = quote.getAsJsonArray("close");
JsonArray volumes = quote.getAsJsonArray("volume");
BarSeries series = new BaseBarSeriesBuilder().withName(ticker).build();
int dataLength = timestamps.size();
for (int i = 0; i < dataLength; i++) {
// Skip bars with null values
if (timestamps.get(i).isJsonNull() || opens.get(i).isJsonNull() || highs.get(i).isJsonNull()
|| lows.get(i).isJsonNull() || closes.get(i).isJsonNull()) {
continue;
}
long timestamp = timestamps.get(i).getAsLong();
Instant endTime = Instant.ofEpochSecond(timestamp);
double openValue = opens.get(i).getAsDouble();
double highValue = highs.get(i).getAsDouble();
double lowValue = lows.get(i).getAsDouble();
double closeValue = closes.get(i).getAsDouble();
double volumeValue = volumes.get(i).isJsonNull() ? 0.0 : volumes.get(i).getAsDouble();
series.barBuilder()
.timePeriod(barInterval)
.endTime(endTime)
.openPrice(openValue)
.highPrice(highValue)
.lowPrice(lowValue)
.closePrice(closeValue)
.volume(volumeValue)
.amount(0)
.add();
}
LOG.debug("Successfully loaded {} bars for ticker {}", series.getBarCount(), ticker);
return series;
} catch (Exception e) {
LOG.error("Error parsing Yahoo Finance response for ticker {}: {}", ticker, e.getMessage(), e);
return null;
}
}
/**
* Merges multiple BarSeries into a single BarSeries, removing duplicates and
* sorting chronologically. Uses a TreeMap keyed by timestamp to automatically
* handle deduplication and sorting.
*
* @param chunks list of BarSeries to merge
* @param ticker the ticker symbol (for the merged series name)
* @param barInterval the bar interval
* @return a merged BarSeries
*/
private static BarSeries mergeBarSeries(List<BarSeries> chunks, String ticker, Duration barInterval) {
// Use TreeMap to automatically sort by timestamp and deduplicate
TreeMap<Instant, BarData> barMap = new TreeMap<>();
// Collect all bars from all chunks
for (BarSeries chunk : chunks) {
for (int i = 0; i < chunk.getBarCount(); i++) {
var bar = chunk.getBar(i);
Instant endTime = bar.getEndTime();
// If we already have a bar at this timestamp, keep the first one (or you could
// merge)
barMap.putIfAbsent(endTime, new BarData(bar));
}
}
// Build the merged series
BarSeries merged = new BaseBarSeriesBuilder().withName(ticker).build();
for (BarData barData : barMap.values()) {
merged.barBuilder()
.timePeriod(barInterval)
.endTime(barData.endTime)
.openPrice(barData.open)
.highPrice(barData.high)
.lowPrice(barData.low)
.closePrice(barData.close)
.volume(barData.volume)
.amount(0)
.add();
}
LOG.debug("Merged {} chunks into {} unique bars for ticker {}", chunks.size(), merged.getBarCount(), ticker);
return merged;
}
/**
* Instance method that loads historical OHLCV data for a given ticker symbol
* with a specified number of bars. The end date/time is set to the current
* time, and the start date/time is calculated based on the bar count and
* interval.
*
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
* "ETH-USD")
* @param interval the bar interval (must be one of the supported Yahoo Finance
* intervals)
* @param barCount the number of bars to fetch
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, int barCount) {
return loadSeriesInstance(ticker, interval, barCount, null);
}
/**
* Instance method that loads historical OHLCV data for a given ticker symbol
* with a specified number of bars and optional notes for cache file naming. The
* end date/time is set to the current time, and the start date/time is
* calculated based on the bar count and interval.
*
* @param ticker the ticker symbol (e.g., "AAPL", "MSFT", "BTC-USD",
* "ETH-USD")
* @param interval the bar interval (must be one of the supported Yahoo Finance
* intervals)
* @param barCount the number of bars to fetch
* @param notes optional notes to include in cache filename (for uniqueness,
* e.g., test identifiers)
* @return a BarSeries containing the historical data, or null if the request
* fails
*/
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, int barCount, String notes) {
if (barCount <= 0) {
LOG.error("Bar count must be greater than 0");
return null;
}
Instant endDateTime = Instant.now();
Duration totalDuration = interval.getDuration().multipliedBy(barCount);
Instant startDateTime = endDateTime.minus(totalDuration);
return loadSeriesInstance(ticker, interval, startDateTime, endDateTime, notes);
}
@Override
public BarSeries loadSeries(String ticker, Duration interval, Instant start, Instant end) {
if (ticker == null || ticker.trim().isEmpty()) {
throw new IllegalArgumentException("Ticker cannot be null or empty");
}
if (interval == null || interval.isNegative() || interval.isZero()) {
throw new IllegalArgumentException("Interval must be positive");
}
if (start == null || end == null) {
throw new IllegalArgumentException("Start and end dates cannot be null");
}
if (start.isAfter(end)) {
throw new IllegalArgumentException("Start date must be before or equal to end date");
}
// Map Duration to YahooFinanceInterval
YahooFinanceInterval yfInterval = mapDurationToInterval(interval);
if (yfInterval == null) {
LOG.warn("Unsupported interval duration: {}. Falling back to DAY_1", interval);
yfInterval = YahooFinanceInterval.DAY_1;
}
return loadSeriesInstance(ticker, yfInterval, start, end);
}
@Override
public BarSeries loadSeries(String source) {
if (source == null || source.trim().isEmpty()) {
throw new IllegalArgumentException("Source cannot be null or empty");
}
// Check if it's a cache file path
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
if (source.startsWith(responseCacheDir) || (!sourcePrefix.isEmpty() && source.contains(sourcePrefix))) {
Path cacheFile = Paths.get(source);
if (Files.exists(cacheFile)) {
String cachedResponse = readFromCache(cacheFile);
if (cachedResponse != null) {
// Try to extract ticker from filename
String filename = cacheFile.getFileName().toString();
// Format: {sourceName}-TICKER-INTERVAL-START-END[_NOTES].json
// Remove extension
String baseName = filename.replace(".json", "");
// Notes section is everything after the last underscore that follows the end
// timestamp
// We need to parse: {sourceName}-TICKER-INTERVAL-START-END[_NOTES]
String[] parts = baseName.split("-");
if (parts.length >= 5) {
// Check if last part contains underscore (indicating notes section)
// Format: END or END_NOTES
// Notes section is ignored for parsing, so we just need to extract the ticker
// and interval
String ticker = parts[1];
// Try to determine interval from filename
YahooFinanceInterval interval = YahooFinanceInterval.DAY_1; // Default
try {
interval = parseIntervalFromApiValue(parts[2]);
} catch (IllegalArgumentException e) {
LOG.debug("Could not parse interval from filename, using default: {}", e.getMessage());
}
return parseYahooFinanceResponse(cachedResponse, ticker, interval.getDuration());
}
}
}
}
// If not a cache file, return null (could be extended to parse other formats)
return null;
}
/**
* Maps a Duration to the closest matching YahooFinanceInterval.
*
* @param duration the duration to map
* @return the matching YahooFinanceInterval, or null if no close match is found
*/
private YahooFinanceInterval mapDurationToInterval(Duration duration) {
long seconds = duration.getSeconds();
for (YahooFinanceInterval interval : YahooFinanceInterval.values()) {
if (interval.getDuration().getSeconds() == seconds) {
return interval;
}
}
return null;
}
/**
* Parses a YahooFinanceInterval from its API value string.
*
* @param apiValue the API value (e.g., "1m", "1d", "1wk")
* @return the matching YahooFinanceInterval
* @throws IllegalArgumentException if no matching interval is found
*/
private YahooFinanceInterval parseIntervalFromApiValue(String apiValue) {
for (YahooFinanceInterval interval : YahooFinanceInterval.values()) {
if (interval.getApiValue().equals(apiValue)) {
return interval;
}
}
throw new IllegalArgumentException("Unknown interval API value: " + apiValue);
}
/**
* Instance method that performs the actual loading logic. This method uses the
* instance's HttpClient (which can be injected for testing).
*/
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, Instant startDateTime,
Instant endDateTime) {
return loadSeriesInstance(ticker, interval, startDateTime, endDateTime, null);
}
/**
* Instance method that performs the actual loading logic with optional notes.
* This method uses the instance's HttpClient (which can be injected for
* testing).
*
* @param ticker the ticker symbol
* @param interval the interval
* @param startDateTime the start date/time
* @param endDateTime the end date/time
* @param notes optional notes to include in cache filename (for
* uniqueness)
* @return the BarSeries or null if request fails
*/
public BarSeries loadSeriesInstance(String ticker, YahooFinanceInterval interval, Instant startDateTime,
Instant endDateTime, String notes) {
if (ticker == null || ticker.trim().isEmpty()) {
LOG.error("Ticker symbol cannot be null or empty");
return null;
}
if (startDateTime == null || endDateTime == null) {
LOG.error("Start and end date/time cannot be null");
return null;
}
if (startDateTime.isAfter(endDateTime)) {
LOG.error("Start date/time must be before or equal to end date/time");
return null;
}
Duration requestedRange = Duration.between(startDateTime, endDateTime);
Duration conservativeLimit = this.getConservativeLimit(interval);
// If the requested range exceeds conservative limits, paginate the request
if (requestedRange.compareTo(conservativeLimit) > 0) {
LOG.debug(
"Requested date range ({}) exceeds conservative limit ({}) for interval {}. "
+ "Splitting into multiple requests and combining results.",
requestedRange, conservativeLimit, interval);
return loadSeriesPaginated(ticker, interval, startDateTime, endDateTime, conservativeLimit, notes);
}
// Single request for smaller ranges
return loadSeriesSingleRequest(ticker, interval, startDateTime, endDateTime, notes);
}
/**
* Truncates a timestamp based on the interval to enable cache hits for requests
* within the same time period. This override provides Yahoo Finance-specific
* truncation logic for week and month boundaries.
* <p>
* For example, for 1-day intervals, timestamps are truncated to the start of
* the day. For 15-minute intervals, timestamps are truncated to the start of
* the 15-minute period. For weekly intervals, timestamps are truncated to the
* start of the week (Monday). For monthly intervals, timestamps are truncated
* to the start of the month.
*
* @param instant the timestamp to truncate
* @param interval the interval to use for truncation
* @return the truncated timestamp
*/
/**
* Generates the cache file path for a given request.
*
* @param ticker the ticker symbol
* @param interval the interval
* @param startDateTime the start date/time (will be truncated)
* @param endDateTime the end date/time (will be truncated)
* @param notes optional notes section to append to filename (can be
* null or empty)
* @return the cache file path
*/
private Path getCacheFilePath(String ticker, YahooFinanceInterval interval, Instant startDateTime,
Instant endDateTime, String notes) {
return getCacheFilePath(ticker, startDateTime, endDateTime, interval.getDuration(), notes);
}
/**
* Generates the cache file path for a given request (without notes).
*
* @param ticker the ticker symbol
* @param interval the interval
* @param startDateTime the start date/time (will be truncated)
* @param endDateTime the end date/time (will be truncated)
* @return the cache file path
*/
private Path getCacheFilePath(String ticker, YahooFinanceInterval interval, Instant startDateTime,
Instant endDateTime) {
return getCacheFilePath(ticker, interval, startDateTime, endDateTime, null);
}
/**
* Makes a single API request for the specified date range with optional notes.
* This is used for requests that don't exceed conservative limits. If caching
* is enabled, checks cache first before making the API request.
*
* @param ticker the ticker symbol
* @param interval the interval
* @param startDateTime the start date/time
* @param endDateTime the end date/time
* @param notes optional notes to include in cache filename (for
* uniqueness)
* @return the BarSeries or null if request fails
*/
private BarSeries loadSeriesSingleRequest(String ticker, YahooFinanceInterval interval, Instant startDateTime,
Instant endDateTime, String notes) {
// Check cache first if caching is enabled
if (enableResponseCaching) {
// Try exact match first (with or without notes)
Path cacheFile = getCacheFilePath(ticker, interval, startDateTime, endDateTime, notes);
if (isCacheValid(cacheFile, interval.getDuration(), endDateTime)) {
String cachedResponse = readFromCache(cacheFile);
if (cachedResponse != null) {
LOG.debug("Using cached response for {} ({} to {})", ticker, startDateTime, endDateTime);
return parseYahooFinanceResponse(cachedResponse, ticker, interval.getDuration());
}
}
// Also try without notes (for backward compatibility)
if (notes != null && !notes.trim().isEmpty()) {
Path cacheFileNoNotes = getCacheFilePath(ticker, interval, startDateTime, endDateTime);
if (isCacheValid(cacheFileNoNotes, interval.getDuration(), endDateTime)) {
String cachedResponse = readFromCache(cacheFileNoNotes);
if (cachedResponse != null) {
LOG.debug("Using cached response for {} ({} to {})", ticker, startDateTime, endDateTime);
return parseYahooFinanceResponse(cachedResponse, ticker, interval.getDuration());
}
}
}
}
try {
String encodedTicker = URLEncoder.encode(ticker.trim(), StandardCharsets.UTF_8);
long period1 = startDateTime.getEpochSecond();
long period2 = endDateTime.getEpochSecond();
String url = String.format("%s%s?interval=%s&period1=%d&period2=%d", YAHOO_FINANCE_API_URL, encodedTicker,
interval.getApiValue(), period1, period2);
LOG.trace("Fetching data from Yahoo Finance: {}", url);
HttpRequest request = HttpRequest.newBuilder()
.uri(URI.create(url))
.header("User-Agent", "Mozilla/5.0")
.timeout(Duration.ofSeconds(30))
.GET()
.build();
HttpResponseWrapper<String> response = httpClient.send(request, HttpResponse.BodyHandlers.ofString());
if (response.statusCode() != 200) {
LOG.error("Yahoo Finance API returned status code: {}", response.statusCode());
return null;
}
String responseBody = response.body();
LOG.trace("Response body: {}", responseBody);
// Cache the response if caching is enabled
if (enableResponseCaching) {
Path cacheFile = getCacheFilePath(ticker, interval, startDateTime, endDateTime, notes);
writeToCache(cacheFile, responseBody);
}
return parseYahooFinanceResponse(responseBody, ticker, interval.getDuration());
} catch (IOException | InterruptedException e) {
LOG.error("Error fetching data from Yahoo Finance for ticker {}: {}", ticker, e.getMessage(), e);
return null;
}
}
/**
* Loads data by splitting a large date range into multiple smaller requests
* (pagination). Each chunk respects the conservative limit, and results are
* merged chronologically.
*
* @param ticker the ticker symbol
* @param interval the bar interval
* @param startDateTime the start date/time
* @param endDateTime the end date/time
* @param chunkSize the maximum size for each chunk
* @param notes optional notes to include in cache filename (for
* uniqueness)
* @return a BarSeries containing all merged data, or null if all requests fail
*/
private BarSeries loadSeriesPaginated(String ticker, YahooFinanceInterval interval, Instant startDateTime,
Instant endDateTime, Duration chunkSize, String notes) {
List<BarSeries> chunks = new ArrayList<>();
Instant currentStart = startDateTime;
int requestCount = 0;
// Calculate number of chunks needed
Duration totalRange = Duration.between(startDateTime, endDateTime);
int estimatedChunks = (int) Math.ceil((double) totalRange.toSeconds() / chunkSize.toSeconds());
LOG.trace("Splitting request into approximately {} chunks", estimatedChunks);
while (currentStart.isBefore(endDateTime)) {
// Calculate chunk end time (don't exceed the requested end time)
Instant chunkEnd = currentStart.plus(chunkSize);
if (chunkEnd.isAfter(endDateTime)) {
chunkEnd = endDateTime;
}
requestCount++;
LOG.trace("Fetching chunk {}/? ({} to {})", requestCount, currentStart, chunkEnd);
BarSeries chunk = loadSeriesSingleRequest(ticker, interval, currentStart, chunkEnd, notes);
if (chunk != null && chunk.getBarCount() > 0) {
chunks.add(chunk);
LOG.trace("Successfully loaded chunk {} with {} bars", requestCount, chunk.getBarCount());
} else {
LOG.warn("Chunk {} returned no data or failed", requestCount);
}
// Move to next chunk (start from the end of current chunk)
currentStart = chunkEnd;
// If we've reached the end, break
if (chunkEnd.equals(endDateTime) || !currentStart.isBefore(endDateTime)) {
break;
}
// Add a small delay between requests to avoid rate limiting
try {
Thread.sleep(100); // 100ms delay between requests
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
LOG.warn("Interrupted during pagination delay");
break;
}
}
if (chunks.isEmpty()) {
LOG.error("All paginated requests failed for ticker {}", ticker);
return null;
}
LOG.debug("Successfully fetched {} chunks, merging {} total bars", chunks.size(),
chunks.stream().mapToInt(BarSeries::getBarCount).sum());
return mergeBarSeries(chunks, ticker, interval.getDuration());
}
/**
* Returns the conservative (safe) maximum date range for a given interval.
* These are smaller than the absolute maximums to ensure reliable API
* responses. Used to determine when pagination is needed.
* <p>
* This method is protected to allow subclasses (e.g., in tests) to override the
* conservative limit for testing pagination functionality.
*
* @param interval the bar interval
* @return the conservative maximum date range
*/
protected Duration getConservativeLimit(YahooFinanceInterval interval) {
return switch (interval) {
case MINUTE_1, MINUTE_5, MINUTE_15, MINUTE_30 -> Duration.ofDays(30); // 30 days for intraday (conservative)
case HOUR_1, HOUR_4 -> Duration.ofDays(60); // 60 days for hourly (conservative)
case DAY_1 -> Duration.ofDays(365); // 1 year for daily (conservative)
case WEEK_1, MONTH_1 -> Duration.ofDays(365 * 5); // 5 years for weekly/monthly (conservative)
};
}
/**
* Supported intervals for Yahoo Finance API. These correspond to the intervals
* that Yahoo Finance's chart API supports.
*/
public enum YahooFinanceInterval {
/**
* 1 minute bars
*/
MINUTE_1(Duration.ofMinutes(1), "1m"),
/**
* 5 minute bars
*/
MINUTE_5(Duration.ofMinutes(5), "5m"),
/**
* 15 minute bars
*/
MINUTE_15(Duration.ofMinutes(15), "15m"),
/**
* 30 minute bars
*/
MINUTE_30(Duration.ofMinutes(30), "30m"),
/**
* 1 hour bars
*/
HOUR_1(Duration.ofHours(1), "1h"),
/**
* 4 hour bars
*/
HOUR_4(Duration.ofHours(4), "4h"),
/**
* 1 day bars
*/
DAY_1(Duration.ofDays(1), "1d"),
/**
* 1 week bars
*/
WEEK_1(Duration.ofDays(7), "1wk"),
/**
* 1 month bars
*/
MONTH_1(Duration.ofDays(30), "1mo");
private final Duration duration;
private final String apiValue;
YahooFinanceInterval(Duration duration, String apiValue) {
this.duration = duration;
this.apiValue = apiValue;
}
/**
* Returns the Duration for this interval.
*
* @return the Duration
*/
public Duration getDuration() {
return duration;
}
/**
* Returns the API string value for this interval.
*
* @return the API string value
*/
public String getApiValue() {
return apiValue;
}
}
/**
* Helper class to hold bar data during merging.
*/
private static class BarData {
final Instant endTime;
final double open;
final double high;
final double low;
final double close;
final double volume;
BarData(Bar bar) {
this.endTime = bar.getEndTime();
this.open = bar.getOpenPrice().doubleValue();
this.high = bar.getHighPrice().doubleValue();
this.low = bar.getLowPrice().doubleValue();
this.close = bar.getClosePrice().doubleValue();
this.volume = bar.getVolume().doubleValue();
}
}
}
@@ -0,0 +1,247 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.file;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.BarSeries;
import org.ta4j.core.BaseBarSeriesBuilder;
import ta4jexamples.datasources.BarSeriesDataSource;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
/**
* Abstract base class for file-based BarSeries data sources.
* <p>
* This class provides common infrastructure for file-based data sources,
* including:
* <ul>
* <li>Parameter validation for ticker, interval, and date range</li>
* <li>Filename pattern building based on interval and date range</li>
* <li>Date range filtering for BarSeries</li>
* <li>DateTimeFormatter utilities for filename formatting</li>
* </ul>
* <p>
* Subclasses should implement the file format-specific logic such as:
* <ul>
* <li>Loading and parsing files (CSV, JSON, etc.)</li>
* <li>File searching with format-specific patterns</li>
* <li>Format-specific data transformation</li>
* </ul>
* <p>
* <strong>Example usage:</strong>
*
* <pre>
* public class MyFileDataSource extends AbstractFileBarSeriesDataSource {
* public MyFileDataSource() {
* super("MySource");
* }
*
* // Implement abstract methods for format-specific loading
* }
* </pre>
*
* @since 0.20
*/
public abstract class AbstractFileBarSeriesDataSource implements BarSeriesDataSource {
private static final Logger LOG = LogManager.getLogger(AbstractFileBarSeriesDataSource.class);
/**
* Date format for filenames: yyyyMMdd
*/
protected static final DateTimeFormatter FILENAME_DATE_FORMAT = DateTimeFormatter.ofPattern("yyyyMMdd");
/**
* Date-time format for filenames with hours: yyyyMMddHH
*/
protected static final DateTimeFormatter FILENAME_DATETIME_HOUR_FORMAT = DateTimeFormatter.ofPattern("yyyyMMddHH");
/**
* Date-time format for filenames with minutes: yyyyMMddHHmm
*/
protected static final DateTimeFormatter FILENAME_DATETIME_MINUTE_FORMAT = DateTimeFormatter
.ofPattern("yyyyMMddHHmm");
private final String sourceName;
/**
* Creates a new AbstractFileBarSeriesDataSource with the specified source name.
*
* @param sourceName the source name for building file search patterns (e.g.,
* "Bitstamp", "Coinbase"). Use empty string for generic
* sources without a prefix.
*/
protected AbstractFileBarSeriesDataSource(String sourceName) {
this.sourceName = sourceName != null ? sourceName : "";
}
@Override
public String getSourceName() {
return sourceName;
}
@Override
public BarSeries loadSeries(String ticker, Duration interval, Instant start, Instant end) {
validateParameters(ticker, interval, start, end);
// Build search patterns for filename matching
// Standard pattern:
// {sourcePrefix}{ticker}-{interval}-{startDateTime}_{endDateTime}.{extension}
// DateTime format depends on interval: minutes -> HHmm, hours -> HH, days ->
// date only
DateTimeFormatter dateTimeFormatter = getDateTimeFormatterForInterval(interval);
String startDateTimeStr = start.atZone(ZoneOffset.UTC).format(dateTimeFormatter);
String endDateTimeStr = end.atZone(ZoneOffset.UTC).format(dateTimeFormatter);
// Also prepare date-only format since existing files use this format
String startDateStr = start.atZone(ZoneOffset.UTC).format(FILENAME_DATE_FORMAT);
String endDateStr = end.atZone(ZoneOffset.UTC).format(FILENAME_DATE_FORMAT);
String intervalStr = formatIntervalForFilename(interval);
String sourcePrefix = sourceName.isEmpty() ? "" : sourceName + "-";
// Delegate to subclass for format-specific file searching
BarSeries series = searchAndLoadFile(ticker, intervalStr, sourcePrefix, startDateTimeStr, endDateTimeStr,
startDateStr, endDateStr, interval, start, end);
if (series != null && !series.isEmpty()) {
return series;
}
LOG.debug("No {} file found matching ticker: {}, interval: {}, date range: {} to {}", getFileExtension(),
ticker, interval, start, end);
return null;
}
/**
* Validates the parameters for loading a series.
*
* @param ticker the ticker symbol
* @param interval the bar interval
* @param start the start date/time
* @param end the end date/time
* @throws IllegalArgumentException if any parameter is invalid
*/
protected void validateParameters(String ticker, Duration interval, Instant start, Instant end) {
if (ticker == null || ticker.trim().isEmpty()) {
throw new IllegalArgumentException("Ticker cannot be null or empty");
}
if (interval == null || interval.isNegative() || interval.isZero()) {
throw new IllegalArgumentException("Interval must be positive");
}
if (start == null || end == null) {
throw new IllegalArgumentException("Start and end dates cannot be null");
}
if (start.isAfter(end)) {
throw new IllegalArgumentException("Start date must be before or equal to end date");
}
}
/**
* Searches for and loads a file matching the specified patterns. Subclasses
* should implement this method to handle format-specific file searching logic.
* <p>
* This method is called by
* {@link #loadSeries(String, Duration, Instant, Instant)} with pre-formatted
* date strings and interval strings. Subclasses should try multiple patterns
* (exact match, date-only format, broader patterns) and return the first
* matching file.
*
* @param ticker the ticker symbol (already validated)
* @param intervalStr the formatted interval string (e.g., "PT1D", "PT5M")
* @param sourcePrefix the source prefix (e.g., "Bitstamp-", or empty
* string)
* @param startDateTimeStr the formatted start date-time string
* @param endDateTimeStr the formatted end date-time string
* @param startDateStr the formatted start date string (date only)
* @param endDateStr the formatted end date string (date only)
* @param interval the interval duration
* @param start the start date/time
* @param end the end date/time
* @return the loaded BarSeries, or null if no matching file is found
*/
protected abstract BarSeries searchAndLoadFile(String ticker, String intervalStr, String sourcePrefix,
String startDateTimeStr, String endDateTimeStr, String startDateStr, String endDateStr, Duration interval,
Instant start, Instant end);
/**
* Returns the file extension (without the dot) for this data source. Used in
* log messages and file pattern building.
*
* @return the file extension (e.g., "csv", "json")
*/
protected abstract String getFileExtension();
/**
* Determines the appropriate DateTimeFormatter for filename datetime formatting
* based on the interval. For minute-level intervals, includes hours and
* minutes. For hour-level intervals, includes hours. For day-level intervals,
* uses date only.
*
* @param interval the bar interval
* @return the appropriate DateTimeFormatter
*/
protected DateTimeFormatter getDateTimeFormatterForInterval(Duration interval) {
long seconds = interval.getSeconds();
if (seconds < 3600) {
// Interval is in minutes or seconds - include hours and minutes
return FILENAME_DATETIME_MINUTE_FORMAT;
} else if (seconds < 86400) {
// Interval is in hours - include hours
return FILENAME_DATETIME_HOUR_FORMAT;
} else {
// Interval is in days or longer - date only
return FILENAME_DATE_FORMAT;
}
}
/**
* Formats a Duration as an ISO 8601 interval string for use in filenames.
*
* @param interval the duration to format
* @return the ISO 8601 duration string (e.g., "PT1D", "PT5M", "PT1H")
*/
protected String formatIntervalForFilename(Duration interval) {
long seconds = interval.getSeconds();
if (seconds % 86400 == 0) {
return "PT" + (seconds / 86400) + "D";
} else if (seconds % 3600 == 0) {
return "PT" + (seconds / 3600) + "H";
} else if (seconds % 60 == 0) {
return "PT" + (seconds / 60) + "M";
} else {
return "PT" + seconds + "S";
}
}
/**
* Filters a BarSeries to only include bars within the specified date range.
*
* @param series the series to filter
* @param start the start date (inclusive)
* @param end the end date (inclusive)
* @return a new BarSeries containing only bars within the date range, or null
* if no bars match
*/
protected BarSeries filterSeriesByDateRange(BarSeries series, Instant start, Instant end) {
if (series == null || series.isEmpty()) {
return null;
}
var filteredSeries = new BaseBarSeriesBuilder().withName(series.getName()).build();
for (int i = 0; i < series.getBarCount(); i++) {
var bar = series.getBar(i);
Instant barEnd = bar.getEndTime();
if (!barEnd.isBefore(start) && !barEnd.isAfter(end)) {
filteredSeries.addBar(bar);
}
}
return filteredSeries.isEmpty() ? null : filteredSeries;
}
}
@@ -0,0 +1,430 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.http;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import java.io.IOException;
import java.net.http.HttpClient;
import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
/**
* Abstract base class for HTTP-based BarSeries data sources.
* <p>
* This class provides common infrastructure for HTTP-based data sources,
* including:
* <ul>
* <li>HTTP client management via {@link HttpClientWrapper}</li>
* <li>Response caching to disk for faster subsequent requests</li>
* <li>Cache validation and management</li>
* <li>Common cache file operations</li>
* </ul>
* <p>
* Subclasses should implement the API-specific logic such as:
* <ul>
* <li>Building API-specific URLs and request parameters</li>
* <li>Parsing API-specific response formats</li>
* <li>Defining interval enums and their API values</li>
* <li>Implementing interval-specific timestamp truncation logic</li>
* </ul>
* <p>
* <strong>Example usage:</strong>
*
* <pre>
* public class MyHttpDataSource extends AbstractHttpBarSeriesDataSource {
* public MyHttpDataSource() {
* super(new DefaultHttpClientWrapper(), false);
* }
*
* // Implement abstract methods and API-specific logic
* }
* </pre>
*
* @since 0.20
*/
public abstract class AbstractHttpBarSeriesDataSource implements HttpBarSeriesDataSource {
/**
* Default directory for caching HTTP responses.
*/
public static final String DEFAULT_RESPONSE_CACHE_DIR = "temp/responses";
private static final Logger LOG = LogManager.getLogger(AbstractHttpBarSeriesDataSource.class);
protected final HttpClientWrapper httpClient;
protected final boolean enableResponseCaching;
protected final String responseCacheDir;
/**
* Creates a new AbstractHttpBarSeriesDataSource with the specified
* HttpClientWrapper and caching option, using the default cache directory.
*
* @param httpClient the HttpClientWrapper to use for API requests
* (can be a mock for testing)
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
* @throws IllegalArgumentException if httpClient is null
*/
protected AbstractHttpBarSeriesDataSource(HttpClientWrapper httpClient, boolean enableResponseCaching) {
if (httpClient == null) {
throw new IllegalArgumentException("HttpClientWrapper cannot be null");
}
this.httpClient = httpClient;
this.enableResponseCaching = enableResponseCaching;
this.responseCacheDir = DEFAULT_RESPONSE_CACHE_DIR;
if (enableResponseCaching) {
ensureCacheDirectoryExists();
}
}
/**
* Creates a new AbstractHttpBarSeriesDataSource with the specified
* HttpClientWrapper and custom cache directory. Response caching is
* automatically enabled when a cache directory is specified.
*
* @param httpClient the HttpClientWrapper to use for API requests (can be
* a mock for testing)
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
* @throws IllegalArgumentException if httpClient is null or responseCacheDir is
* null or empty
*/
protected AbstractHttpBarSeriesDataSource(HttpClientWrapper httpClient, String responseCacheDir) {
if (httpClient == null) {
throw new IllegalArgumentException("HttpClientWrapper cannot be null");
}
if (responseCacheDir == null || responseCacheDir.trim().isEmpty()) {
throw new IllegalArgumentException("Response cache directory cannot be null or empty");
}
this.httpClient = httpClient;
this.enableResponseCaching = true;
this.responseCacheDir = responseCacheDir.trim();
ensureCacheDirectoryExists();
}
/**
* Creates a new AbstractHttpBarSeriesDataSource with the specified HttpClient.
* This is a convenience constructor that wraps the HttpClient in a
* DefaultHttpClientWrapper, using the default cache directory.
*
* @param httpClient the HttpClient to use for API requests
* @param enableResponseCaching if true, responses will be cached to disk for
* faster subsequent requests
*/
protected AbstractHttpBarSeriesDataSource(HttpClient httpClient, boolean enableResponseCaching) {
this(new DefaultHttpClientWrapper(httpClient), enableResponseCaching);
}
/**
* Creates a new AbstractHttpBarSeriesDataSource with the specified HttpClient
* and custom cache directory. This is a convenience constructor that wraps the
* HttpClient in a DefaultHttpClientWrapper. Response caching is automatically
* enabled when a cache directory is specified.
*
* @param httpClient the HttpClient to use for API requests
* @param responseCacheDir the directory path for caching responses (can be
* relative or absolute)
*/
protected AbstractHttpBarSeriesDataSource(HttpClient httpClient, String responseCacheDir) {
this(new DefaultHttpClientWrapper(httpClient), responseCacheDir);
}
@Override
public HttpClientWrapper getHttpClient() {
return httpClient;
}
@Override
public boolean isResponseCachingEnabled() {
return enableResponseCaching;
}
/**
* Returns the response cache directory path.
*
* @return the cache directory path
*/
public String getResponseCacheDir() {
return responseCacheDir;
}
/**
* Ensures the cache directory exists, creating it if necessary.
*/
protected void ensureCacheDirectoryExists() {
try {
Path cacheDir = Paths.get(responseCacheDir);
if (!Files.exists(cacheDir)) {
Files.createDirectories(cacheDir);
LOG.debug("Created cache directory: {}", cacheDir.toAbsolutePath());
}
} catch (IOException e) {
LOG.warn("Failed to create cache directory: {}", e.getMessage());
}
}
/**
* Truncates a timestamp based on the interval duration to enable cache hits for
* requests within the same time period. This is a generic implementation that
* works with Duration. Subclasses can override this method to provide more
* precise truncation logic for specific interval types (e.g., week boundaries,
* month boundaries).
* <p>
* For example, for 1-day intervals, timestamps are truncated to the start of
* the day. For 15-minute intervals, timestamps are truncated to the start of
* the 15-minute period.
*
* @param instant the timestamp to truncate
* @param interval the interval duration to use for truncation
* @return the truncated timestamp
*/
protected Instant truncateTimestampForCache(Instant instant, Duration interval) {
ZonedDateTime zdt = instant.atZone(ZoneOffset.UTC);
ZonedDateTime truncated;
long seconds = interval.getSeconds();
long minutes = seconds / 60;
long hours = minutes / 60;
long days = hours / 24;
if (days >= 30) {
// For monthly intervals (approximately 30 days), truncate to start of month
truncated = zdt.withDayOfMonth(1).withHour(0).withMinute(0).withSecond(0).withNano(0);
} else if (days >= 7) {
// For weekly intervals (approximately 7 days), truncate to start of week
// (Monday)
int daysFromMonday = zdt.getDayOfWeek().getValue() - java.time.DayOfWeek.MONDAY.getValue();
if (daysFromMonday < 0) {
daysFromMonday += 7; // Handle Sunday (value 7)
}
truncated = zdt.minusDays(daysFromMonday).withHour(0).withMinute(0).withSecond(0).withNano(0);
} else if (days >= 1) {
// For daily intervals, truncate to start of day
truncated = zdt.withHour(0).withMinute(0).withSecond(0).withNano(0);
} else if (hours >= 1) {
// For hourly intervals, truncate to start of hour period
int hour = (int) ((zdt.getHour() / hours) * hours);
truncated = zdt.withHour(hour).withMinute(0).withSecond(0).withNano(0);
} else if (minutes >= 1) {
// For minute intervals, truncate to start of minute period
int minute = (int) ((zdt.getMinute() / minutes) * minutes);
truncated = zdt.withMinute(minute).withSecond(0).withNano(0);
} else {
// For second-level intervals, truncate to start of second
int second = (int) ((zdt.getSecond() / seconds) * seconds);
truncated = zdt.withSecond(second).withNano(0);
}
return truncated.toInstant();
}
/**
* Generates the cache file path for a given request.
*
* @param symbol the symbol (ticker, product ID, etc.)
* @param startDateTime the start date/time (will be truncated)
* @param endDateTime the end date/time (will be truncated)
* @param interval the interval duration (used for truncation and filename)
* @param notes optional notes section to append to filename (can be
* null or empty)
* @return the cache file path
*/
protected Path getCacheFilePath(String symbol, Instant startDateTime, Instant endDateTime, Duration interval,
String notes) {
Instant truncatedStart = truncateTimestampForCache(startDateTime, interval);
Instant truncatedEnd = truncateTimestampForCache(endDateTime, interval);
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
String notesSuffix = (notes != null && !notes.trim().isEmpty()) ? "_" + notes.trim() : "";
// Use Duration.toString() for standardized format (e.g., "PT24H" for 1 day,
// "PT1H" for 1 hour)
String durationString = interval.toString();
String filename = String.format("%s%s-%s-%d-%d%s.json", sourcePrefix,
symbol.toUpperCase().replaceAll("[^A-Z0-9-]", "_"), durationString, truncatedStart.getEpochSecond(),
truncatedEnd.getEpochSecond(), notesSuffix);
return Paths.get(responseCacheDir, filename);
}
/**
* Checks if a cache file exists and is still valid based on the interval. For
* historical data (end date in the past), cache is valid indefinitely. For
* current data (end date is recent), cache expires after the interval duration.
* <p>
* Cache validity is determined by the file's modification time (when the cache
* file was created), not access time. This is appropriate because we want to
* know how old the cached data is, not when it was last read.
*
* @param cacheFile the cache file path
* @param interval the interval duration
* @param endDateTime the end date/time of the request
* @return true if cache is valid, false otherwise
*/
protected boolean isCacheValid(Path cacheFile, Duration interval, Instant endDateTime) {
if (!Files.exists(cacheFile)) {
return false;
}
try {
// Check file modification time (when the cache file was created/written)
Instant fileModified = Files.getLastModifiedTime(cacheFile).toInstant();
Instant now = Instant.now();
// If the request is for historical data (end date is in the past), cache is
// valid indefinitely
if (endDateTime.isBefore(now.minus(interval))) {
return true;
}
// For current/recent data, cache expires after the interval duration
Duration age = Duration.between(fileModified, now);
return age.compareTo(interval) < 0;
} catch (IOException e) {
LOG.warn("Failed to check cache file validity: {}", e.getMessage());
return false;
}
}
/**
* Reads a cached response from disk.
*
* @param cacheFile the cache file path
* @return the cached JSON response, or null if read fails
*/
protected String readFromCache(Path cacheFile) {
try {
String content = Files.readString(cacheFile, StandardCharsets.UTF_8);
LOG.debug("Cache hit: {}", cacheFile.getFileName());
return content;
} catch (IOException e) {
LOG.warn("Failed to read from cache: {}", e.getMessage());
return null;
}
}
/**
* Writes a response to the cache.
*
* @param cacheFile the cache file path
* @param response the JSON response to cache
*/
protected void writeToCache(Path cacheFile, String response) {
try {
// Ensure parent directory exists
Path parentDir = cacheFile.getParent();
if (parentDir != null && !Files.exists(parentDir)) {
Files.createDirectories(parentDir);
}
Files.writeString(cacheFile, response, StandardCharsets.UTF_8);
LOG.debug("Cached response: {}", cacheFile.getFileName());
} catch (IOException e) {
LOG.warn("Failed to write to cache: {}", e.getMessage());
}
}
/**
* Deletes all cache files in the cache directory that belong to this data
* source. Files are identified by the source name prefix (e.g., "Coinbase-",
* "YahooFinance-").
*
* @return the number of files deleted
*/
public int deleteAllCacheFiles() {
return deleteCacheFilesOlderThan(Duration.ZERO);
}
/**
* Deletes cache files that are older than the specified maximum age. Files are
* identified by the source name prefix (e.g., "Coinbase-", "YahooFinance-") and
* are deleted based on their file modification time (when the cache file was
* created/written).
* <p>
* <strong>Note:</strong> This method uses file modification time, not access
* time. Modification time represents when the cache file was created (when the
* response was cached), which is appropriate for determining cache age. Access
* time (when the file was last read) is not used because it's not reliably
* available across all platforms and filesystems.
*
* @param maxAge the maximum age of files to keep (files older than this will be
* deleted). Use {@link Duration#ZERO} to delete all files.
* @return the number of files deleted
*/
public int deleteCacheFilesOlderThan(Duration maxAge) {
if (!Files.exists(Paths.get(responseCacheDir))) {
return 0;
}
String sourcePrefix = getSourceName().isEmpty() ? "" : getSourceName() + "-";
int deletedCount = 0;
Instant cutoffTime = Instant.now().minus(maxAge);
try {
Path cacheDir = Paths.get(responseCacheDir);
for (Path path : Files.list(cacheDir).toList()) {
if (Files.isRegularFile(path)) {
String filename = path.getFileName().toString();
// Check if file belongs to this data source
if (sourcePrefix.isEmpty() || filename.startsWith(sourcePrefix)) {
try {
// If maxAge is ZERO, delete all files regardless of age
if (maxAge.isZero() || Files.getLastModifiedTime(path).toInstant().isBefore(cutoffTime)) {
Files.delete(path);
deletedCount++;
LOG.debug("Deleted cache file: {}", filename);
}
} catch (IOException e) {
LOG.warn("Failed to delete cache file {}: {}", filename, e.getMessage());
}
}
}
}
} catch (IOException e) {
LOG.warn("Failed to list cache directory: {}", e.getMessage());
return deletedCount;
}
if (deletedCount > 0) {
LOG.info("Deleted {} stale cache file(s) with prefix '{}'", deletedCount, sourcePrefix);
}
return deletedCount;
}
/**
* Deletes stale cache files. A file is considered stale if it's older than 30
* days and the end timestamp encoded in the filename indicates it's for
* historical data (data that won't change). This is a conservative approach
* that preserves recent caches and historical data caches that might still be
* useful.
* <p>
* For more control, use {@link #deleteCacheFilesOlderThan(Duration)} or
* {@link #deleteAllCacheFiles()}.
*
* @return the number of files deleted
*/
public int deleteStaleCacheFiles() {
return deleteStaleCacheFiles(Duration.ofDays(30));
}
/**
* Deletes stale cache files older than the specified age. A file is considered
* stale if it's older than the specified age. Files are identified by the
* source name prefix (e.g., "Coinbase-", "YahooFinance-").
*
* @param maxAge the maximum age of files to keep (files older than this will be
* deleted)
* @return the number of files deleted
*/
public int deleteStaleCacheFiles(Duration maxAge) {
return deleteCacheFilesOlderThan(maxAge);
}
}
@@ -0,0 +1,45 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.http;
import java.io.IOException;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
import java.time.Duration;
/**
* Default implementation of {@link HttpClientWrapper} that delegates to a
* {@link HttpClient}.
*/
public class DefaultHttpClientWrapper implements HttpClientWrapper {
private final HttpClient httpClient;
/**
* Creates a new wrapper around the specified HttpClient.
*
* @param httpClient the HttpClient to wrap
*/
public DefaultHttpClientWrapper(HttpClient httpClient) {
if (httpClient == null) {
throw new IllegalArgumentException("HttpClient cannot be null");
}
this.httpClient = httpClient;
}
/**
* Creates a new wrapper with a default HttpClient.
*/
public DefaultHttpClientWrapper() {
this(HttpClient.newBuilder().connectTimeout(Duration.ofSeconds(10)).build());
}
@Override
public <T> HttpResponseWrapper<T> send(HttpRequest request, HttpResponse.BodyHandler<T> handler)
throws IOException, InterruptedException {
HttpResponse<T> response = httpClient.send(request, handler);
return new DefaultHttpResponseWrapper<>(response);
}
}
@@ -0,0 +1,39 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.http;
import java.net.http.HttpResponse;
/**
* Default implementation of {@link HttpResponseWrapper} that delegates to a
* {@link HttpResponse}.
*
* @param <T> the response body type
*/
public class DefaultHttpResponseWrapper<T> implements HttpResponseWrapper<T> {
private final HttpResponse<T> httpResponse;
/**
* Creates a new wrapper around the specified HttpResponse.
*
* @param httpResponse the HttpResponse to wrap
*/
public DefaultHttpResponseWrapper(HttpResponse<T> httpResponse) {
if (httpResponse == null) {
throw new IllegalArgumentException("HttpResponse cannot be null");
}
this.httpResponse = httpResponse;
}
@Override
public int statusCode() {
return httpResponse.statusCode();
}
@Override
public T body() {
return httpResponse.body();
}
}
@@ -0,0 +1,46 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.http;
import ta4jexamples.datasources.BarSeriesDataSource;
/**
* Marker interface for HTTP-based BarSeries data sources.
* <p>
* This interface extends {@link BarSeriesDataSource} to identify data sources
* that fetch data over HTTP. Implementations of this interface typically use
* {@link HttpClientWrapper} for making HTTP requests and may support response
* caching.
* <p>
* HTTP-based data sources share common functionality such as:
* <ul>
* <li>HTTP request handling via {@link HttpClientWrapper}</li>
* <li>Response caching to disk for faster subsequent requests</li>
* <li>Cache validation and management</li>
* <li>Pagination for large data requests</li>
* </ul>
* <p>
* For shared implementation, consider extending
* {@link AbstractHttpBarSeriesDataSource} which provides common HTTP and
* caching infrastructure.
*
* @since 0.20
*/
public interface HttpBarSeriesDataSource extends BarSeriesDataSource {
/**
* Returns the HttpClientWrapper used by this data source for making HTTP
* requests.
*
* @return the HttpClientWrapper instance
*/
HttpClientWrapper getHttpClient();
/**
* Returns whether response caching is enabled for this data source.
*
* @return true if caching is enabled, false otherwise
*/
boolean isResponseCachingEnabled();
}
@@ -0,0 +1,32 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.http;
import java.io.IOException;
import java.net.http.HttpRequest;
/**
* Wrapper around {@link java.net.http.HttpClient} to enable easier testing via
* mocking.
* <p>
* This wrapper provides a simple interface for HTTP operations that can be
* easily mocked in unit tests, avoiding the need to mock the final
* {@link java.net.http.HttpClient} class directly.
*/
public interface HttpClientWrapper {
/**
* Sends the given request using this client, blocking if necessary to get the
* response.
*
* @param <T> the response body type
* @param request the request
* @param handler the response body handler
* @return the response wrapper
* @throws IOException if an I/O error occurs when sending or receiving
* @throws InterruptedException if the operation is interrupted
*/
<T> HttpResponseWrapper<T> send(HttpRequest request, java.net.http.HttpResponse.BodyHandler<T> handler)
throws IOException, InterruptedException;
}
@@ -0,0 +1,31 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.http;
/**
* Wrapper around {@link java.net.http.HttpResponse} to enable easier testing
* via mocking.
* <p>
* This wrapper provides a simple interface for HTTP response operations that
* can be easily mocked in unit tests, avoiding the need to mock the final
* {@link java.net.http.HttpResponse} class directly.
*
* @param <T> the response body type
*/
public interface HttpResponseWrapper<T> {
/**
* Returns the status code for this response.
*
* @return the status code
*/
int statusCode();
/**
* Returns the body of this response.
*
* @return the response body
*/
T body();
}
@@ -0,0 +1,316 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.json;
import com.google.gson.*;
import com.google.gson.stream.JsonReader;
import com.google.gson.stream.JsonWriter;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.BarSeries;
import org.ta4j.core.BaseBarSeries;
import org.ta4j.core.BaseBarSeriesBuilder;
import java.io.IOException;
import java.time.Duration;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
/**
* A TypeAdapter implementation for deserializing JSON data into BarSeries
* objects. This adapter supports multiple JSON formats by detecting the
* structure of the input data. It currently handles two formats:
* <ul>
* <li><strong>Coinbase format:</strong> identified by the presence of a
* "candles" array. The "start" field represents the start of the candle period,
* and the end time is calculated as start + duration. This adapter can be used
* for both Coinbase API responses (where the interval is known) and Coinbase
* JSON files (where the interval can be inferred from the data).</li>
* <li><strong>Binance format:</strong> identified by the presence of an "ohlc"
* array</li>
* </ul>
* <p>
* The adapter parses the JSON input and converts it into a BaseBarSeries
* instance populated with the appropriate bar data. The bar data is sorted by
* timestamp for the Coinbase format to ensure chronological order.
* <p>
* For Coinbase format, the adapter provides a static helper method
* {@link #parseCoinbaseFormat(JsonObject, String, Duration)} that can be used
* directly when parsing Coinbase data, allowing you to specify a known interval
* (for API responses) or let it be inferred (for JSON files).
* <p>
* Write operations are not supported by this adapter and will throw an
* exception.
* <p>
* The adapter uses internal lightweight data classes to temporarily hold the
* parsed JSON data before converting it into the final BarSeries format.
* <p>
* Logging is implemented to track which format is being parsed during
* deserialization.
*
* @since 0.19
*/
public class AdaptiveBarSeriesTypeAdapter extends TypeAdapter<BarSeries> {
private static final Logger LOG = LogManager.getLogger(AdaptiveBarSeriesTypeAdapter.class);
@Override
public void write(JsonWriter out, BarSeries value) throws IOException {
// Not implemented for this use case
throw new UnsupportedOperationException("Write operation not supported");
}
/**
* Reads a BarSeries from the provided JsonReader by detecting the format based
* on available fields. The method parses the JSON input and delegates to
* appropriate parsing methods depending on whether the root object contains
* "candles" or "ohlc" fields.
*
* @param in the JsonReader to read the JSON data from
* @return the parsed BarSeries object
* @throws JsonParseException if the JSON format is unknown or neither "candles"
* nor "ohlc" fields are found
* @since 0.19
*/
@Override
public BarSeries read(JsonReader in) {
JsonElement json = JsonParser.parseReader(in);
JsonObject root = json.getAsJsonObject();
// Detect format based on available fields
if (root.has("candles")) {
return parseCoinbaseFormat(root);
} else if (root.has("ohlc")) {
return parseBinanceFormat(root);
} else {
throw new JsonParseException("Unknown format - neither 'candles' nor 'ohlc' found");
}
}
/**
* Parses a JSON object in Coinbase format into a BarSeries. The input JSON is
* expected to contain a "candles" array where each element represents a bar
* with start time, open, high, low, close, and volume values.
* <p>
* Note: Coinbase API's "start" field represents the start of the time interval
* for the candle. The end time is calculated as start + duration. This method
* infers the duration from the data. For cases where the interval is known, use
* {@link #parseCoinbaseFormat(JsonObject, String, Duration)} instead.
*
* @param root the JsonObject representing the root of the JSON data in Coinbase
* format
* @return a BarSeries populated with data parsed from the Coinbase format JSON
*/
private BarSeries parseCoinbaseFormat(JsonObject root) {
return parseCoinbaseFormat(root, "CoinbaseData", null);
}
/**
* Parses a JSON object in Coinbase format into a BarSeries with a known
* interval and series name. The input JSON is expected to contain a "candles"
* array where each element represents a bar with start time, open, high, low,
* close, and volume values.
* <p>
* Note: Coinbase API's "start" field represents the start of the time interval
* for the candle. The end time is calculated as start + duration. This method
* uses the provided interval directly instead of inferring it from the data.
* <p>
* This method can be used for both API responses (where the interval is known)
* and JSON files (where the interval can be inferred or provided).
*
* @param root the JsonObject representing the root of the JSON data in
* Coinbase format
* @param seriesName the name to use for the BarSeries
* @param knownInterval the known bar interval (if null, will be inferred from
* the data by calculating the difference between
* consecutive start times)
* @return a BarSeries populated with data parsed from the Coinbase format JSON
*/
public static BarSeries parseCoinbaseFormat(JsonObject root, String seriesName, Duration knownInterval) {
LOG.trace("Parsing Coinbase format");
JsonArray candles = root.getAsJsonArray("candles");
if (candles == null || candles.isEmpty()) {
return new BaseBarSeriesBuilder().withName(seriesName).build();
}
List<CoinbaseBar> barList = new ArrayList<>();
for (JsonElement candle : candles) {
JsonObject candleObj = candle.getAsJsonObject();
// Skip candles with null or missing required fields
if (candleObj.get("start") == null || candleObj.get("start").isJsonNull() || candleObj.get("open") == null
|| candleObj.get("open").isJsonNull() || candleObj.get("high") == null
|| candleObj.get("high").isJsonNull() || candleObj.get("low") == null
|| candleObj.get("low").isJsonNull() || candleObj.get("close") == null
|| candleObj.get("close").isJsonNull()) {
continue;
}
// Handle null volume by defaulting to "0"
String volume = "0";
if (candleObj.get("volume") != null && !candleObj.get("volume").isJsonNull()) {
volume = candleObj.get("volume").getAsString();
}
// Validate timestamp format before creating CoinbaseBar
String startStr = candleObj.get("start").getAsString();
try {
Long.parseLong(startStr);
} catch (NumberFormatException nfe) {
LOG.warn("Invalid timestamp format in Coinbase data, skipping candle: {}", startStr, nfe);
continue;
}
barList.add(
new CoinbaseBar(startStr, candleObj.get("open").getAsString(), candleObj.get("high").getAsString(),
candleObj.get("low").getAsString(), candleObj.get("close").getAsString(), volume));
}
// Sort by timestamp (ascending order)
barList.sort(Comparator.comparingLong(CoinbaseBar::getStartTime));
// Build series
BaseBarSeries series = new BaseBarSeriesBuilder().withName(seriesName).build();
Duration lastDuration = knownInterval;
for (int i = 0; i < barList.size(); i++) {
CoinbaseBar bar = barList.get(i);
// Coinbase "start" field is the start of the candle period
Instant startTime = bar.getStartInstant();
Duration duration;
if (knownInterval != null) {
duration = knownInterval;
} else {
// Infer duration from data by calculating the difference between consecutive
// start times
Instant previousStart = i > 0 ? barList.get(i - 1).getStartInstant() : null;
Instant currentStart = bar.getStartInstant();
Instant nextStart = i + 1 < barList.size() ? barList.get(i + 1).getStartInstant() : null;
duration = inferDuration(previousStart, currentStart, nextStart, lastDuration);
lastDuration = duration;
}
// End time is start time + duration
Instant endTime = startTime.plus(duration);
bar.addToSeries(series, endTime, duration);
}
return series;
}
/**
* Parses a JSON object in Binance format into a BarSeries. The input JSON is
* expected to contain an "ohlc" array where each element represents a bar with
* end time, open price, high price, low price, close price, volume, and amount
* values.
*
* @param root the JsonObject representing the root of the JSON data in Binance
* format
* @return a BarSeries populated with data parsed from the Binance format JSON
*/
private BarSeries parseBinanceFormat(JsonObject root) {
LOG.trace("Parsing Binance format");
JsonArray ohlc = root.getAsJsonArray("ohlc");
String seriesName = root.has("name") ? root.get("name").getAsString() : "BinanceData";
BaseBarSeries series = new BaseBarSeriesBuilder().withName(seriesName).build();
List<BinanceBar> bars = new ArrayList<>();
for (JsonElement barElement : ohlc) {
JsonObject barObj = barElement.getAsJsonObject();
BinanceBar bar = new BinanceBar(barObj.get("endTime").getAsLong(), barObj.get("openPrice").getAsNumber(),
barObj.get("highPrice").getAsNumber(), barObj.get("lowPrice").getAsNumber(),
barObj.get("closePrice").getAsNumber(), barObj.get("volume").getAsNumber(),
barObj.get("amount").getAsNumber());
bars.add(bar);
}
bars.sort(Comparator.comparingLong(BinanceBar::endTime));
Duration lastDuration = null;
for (int i = 0; i < bars.size(); i++) {
BinanceBar bar = bars.get(i);
Instant previousEnd = i > 0 ? bars.get(i - 1).getEndInstant() : null;
Instant currentEnd = bar.getEndInstant();
Instant nextEnd = i + 1 < bars.size() ? bars.get(i + 1).getEndInstant() : null;
Duration duration = inferDuration(previousEnd, currentEnd, nextEnd, lastDuration);
lastDuration = duration;
bar.addToSeries(series, currentEnd, duration);
}
return series;
}
private static Duration inferDuration(Instant previous, Instant current, Instant next, Duration fallback) {
Duration candidate = null;
if (next != null) {
candidate = Duration.between(current, next);
} else if (previous != null) {
candidate = Duration.between(previous, current);
}
if (candidate != null) {
if (candidate.isNegative()) {
candidate = candidate.negated();
}
if (!candidate.isZero()) {
return candidate;
}
}
if (fallback != null && !fallback.isZero() && !fallback.isNegative()) {
return fallback;
}
return Duration.ofSeconds(1);
}
// Lightweight data classes for internal use only
private record CoinbaseBar(String start, String open, String high, String low, String close, String volume) {
public long getStartTime() {
return Long.parseLong(start);
}
public Instant getStartInstant() {
return Instant.ofEpochSecond(getStartTime());
}
public void addToSeries(BaseBarSeries series, Instant endTime, Duration timePeriod) {
series.barBuilder()
.timePeriod(timePeriod)
.endTime(endTime)
.openPrice(open)
.highPrice(high)
.lowPrice(low)
.closePrice(close)
.volume(volume)
.add();
}
}
private record BinanceBar(long endTime, Number openPrice, Number highPrice, Number lowPrice, Number closePrice,
Number volume, Number amount) {
public Instant getEndInstant() {
return Instant.ofEpochMilli(endTime);
}
public void addToSeries(BaseBarSeries series, Instant endTimeInstant, Duration timePeriod) {
series.barBuilder()
.timePeriod(timePeriod)
.endTime(endTimeInstant)
.openPrice(openPrice)
.highPrice(highPrice)
.lowPrice(lowPrice)
.closePrice(closePrice)
.volume(volume)
.amount(amount)
.add();
}
}
}
@@ -0,0 +1,59 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.json;
import org.ta4j.core.Bar;
import org.ta4j.core.BaseBarSeries;
import org.ta4j.core.utils.DeprecationNotifier;
import java.time.Duration;
import java.time.Instant;
@Deprecated(since = "0.19", forRemoval = true)
public class GsonBarData {
private long endTime;
private Number openPrice;
private Number highPrice;
private Number lowPrice;
private Number closePrice;
private Number volume;
private Number amount;
public GsonBarData() {
DeprecationNotifier.warnOnce(GsonBarData.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
"0.24.0");
}
@Deprecated(since = "0.19", forRemoval = true)
public static GsonBarData from(Bar bar) {
DeprecationNotifier.warnOnce(GsonBarData.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
"0.24.0");
var result = new GsonBarData();
result.endTime = bar.getEndTime().toEpochMilli();
result.openPrice = bar.getOpenPrice().getDelegate();
result.highPrice = bar.getHighPrice().getDelegate();
result.lowPrice = bar.getLowPrice().getDelegate();
result.closePrice = bar.getClosePrice().getDelegate();
result.volume = bar.getVolume().getDelegate();
result.amount = bar.getAmount().getDelegate();
return result;
}
@Deprecated(since = "0.19", forRemoval = true)
public void addTo(BaseBarSeries barSeries) {
DeprecationNotifier.warnOnce(GsonBarData.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
"0.24.0");
var endTimeInstant = Instant.ofEpochMilli(endTime);
barSeries.barBuilder()
.timePeriod(Duration.ofDays(1))
.endTime(endTimeInstant)
.openPrice(openPrice)
.highPrice(highPrice)
.lowPrice(lowPrice)
.closePrice(closePrice)
.volume(volume)
.amount(amount)
.add();
}
}
@@ -0,0 +1,50 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.json;
import org.ta4j.core.Bar;
import org.ta4j.core.BarSeries;
import org.ta4j.core.BaseBarSeries;
import org.ta4j.core.BaseBarSeriesBuilder;
import org.ta4j.core.utils.DeprecationNotifier;
import java.util.LinkedList;
import java.util.List;
@Deprecated(since = "0.19", forRemoval = true)
public class GsonBarSeries {
private String name;
private List<GsonBarData> ohlc = new LinkedList<>();
public GsonBarSeries() {
DeprecationNotifier.warnOnce(GsonBarSeries.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
"0.24.0");
}
@Deprecated(since = "0.19", forRemoval = true)
public static GsonBarSeries from(BarSeries series) {
DeprecationNotifier.warnOnce(GsonBarSeries.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
"0.24.0");
GsonBarSeries result = new GsonBarSeries();
result.name = series.getName();
List<Bar> barData = series.getBarData();
for (Bar bar : barData) {
GsonBarData exportableBarData = GsonBarData.from(bar);
result.ohlc.add(exportableBarData);
}
return result;
}
@Deprecated(since = "0.19", forRemoval = true)
public BarSeries toBarSeries() {
DeprecationNotifier.warnOnce(GsonBarSeries.class, "ta4jexamples.datasources.json.JsonFileBarSeriesDataSource",
"0.24.0");
BaseBarSeries result = new BaseBarSeriesBuilder().withName(this.name).build();
for (GsonBarData data : ohlc) {
data.addTo(result);
}
return result;
}
}
@@ -0,0 +1,135 @@
/*
* SPDX-License-Identifier: MIT
*/
package ta4jexamples.datasources.json;
import com.google.gson.Gson;
import com.google.gson.GsonBuilder;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.ta4j.core.BarSeries;
import org.ta4j.core.utils.DeprecationNotifier;
import java.io.*;
/**
* @deprecated
* <p>
* Use {@link JsonFileBarSeriesDataSource} instead. Scheduled for
* removal in 0.24.0.
* </p>
*/
@Deprecated(since = "0.19", forRemoval = true)
public class JsonBarsSerializer {
private static final Logger LOG = LogManager.getLogger(JsonBarsSerializer.class);
public JsonBarsSerializer() {
warnDeprecatedUse();
}
private static void warnDeprecatedUse() {
DeprecationNotifier.warnOnce(JsonBarsSerializer.class,
"ta4jexamples.datasources.json.JsonFileBarSeriesDataSource", "0.24.0");
}
@Deprecated(since = "0.19", forRemoval = true)
public static void persistSeries(BarSeries series, String filename) {
warnDeprecatedUse();
GsonBarSeries exportableSeries = GsonBarSeries.from(series);
Gson gson = new GsonBuilder().setPrettyPrinting().create();
FileWriter writer = null;
try {
writer = new FileWriter(filename);
gson.toJson(exportableSeries, writer);
LOG.debug("Bar series '{}' successfully saved to '{}'", series.getName(), filename);
} catch (IOException e) {
LOG.error("Unable to store bars in JSON", e);
} finally {
if (writer != null) {
try {
writer.flush();
writer.close();
} catch (IOException e) {
LOG.warn("Unable to flush and/or close file writer", e);
}
}
}
}
@Deprecated(since = "0.19", forRemoval = true)
public static BarSeries loadSeries(String filename) {
warnDeprecatedUse();
Gson gson = new Gson();
FileReader reader = null;
BarSeries result = null;
try {
reader = new FileReader(filename);
GsonBarSeries loadedSeries = gson.fromJson(reader, GsonBarSeries.class);
result = loadedSeries.toBarSeries();
LOG.debug("Bar series '" + result.getName() + "' successfully loaded. #Entries: " + result.getBarCount());
} catch (FileNotFoundException e) {
LOG.error("Unable to load bars from JSON", e);
} finally {
try {
if (reader != null)
reader.close();
} catch (IOException e) {
LOG.warn("Unable to close file reader", e);
}
}
return result;
}
/**
* Loads a BarSeries from the provided InputStream containing JSON data. The
* method parses the JSON content using Gson library and converts it to a
* BarSeries object. If the input stream is null or parsing fails, appropriate
* warning or error messages are logged and null is returned.
*
* @deprecated
* <p>
* Use {@link JsonFileBarSeriesDataSource#loadSeries(String)}
* instead.
*
* @param inputStream the input stream containing JSON data to be parsed into a
* BarSeries
* @return the loaded BarSeries object, or null if loading fails or input stream
* is null
*/
@Deprecated(since = "0.19", forRemoval = true)
public static BarSeries loadSeries(InputStream inputStream) {
warnDeprecatedUse();
if (inputStream == null) {
LOG.warn("Input stream is null, returning null");
return null;
}
Gson gson = new Gson();
InputStreamReader reader = null;
BarSeries result = null;
try {
reader = new InputStreamReader(inputStream);
GsonBarSeries loadedSeries = gson.fromJson(reader, GsonBarSeries.class);
if (loadedSeries == null) {
LOG.warn("Failed to parse JSON, loadedSeries is null");
return null;
}
result = loadedSeries.toBarSeries();
LOG.debug("Bar series '" + result.getName() + "' successfully loaded. #Entries: " + result.getBarCount());
} catch (Exception e) {
LOG.error("Unable to load bars from JSON", e);
} finally {
try {
if (reader != null)
reader.close();
} catch (IOException e) {
LOG.warn("Error closing input stream reader", e);
}
}
return result;
}
}