From 36bca66160cc5d57661c0f1441dd9e307f75c56f Mon Sep 17 00:00:00 2001 From: Martin Oberhuber < martin.oberhuber@windriver.com> Date: Wed, 10 Jun 2009 23:45:40 +0000 Subject: [PATCH] [279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252 --- .../dstore/files/DStoreFileService.java | 42 ++++++++++++++- .../DefaultFileServiceCodePageConverter.java | 53 +++++++++++++------ .../files/IFileServiceCodePageConverter.java | 27 +++++----- 3 files changed, 91 insertions(+), 31 deletions(-) diff --git a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java index 176d3fe2aa9..616fe0f01bf 100644 --- a/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java +++ b/rse/plugins/org.eclipse.rse.services.dstore/src/org/eclipse/rse/internal/services/dstore/files/DStoreFileService.java @@ -56,6 +56,7 @@ * David McKnight (IBM) - [270468] [dstore] FileServiceSubSystem.list() returns folders when only FILE_TYPE_FILES is requested * David McKnight (IBM) - [272335] [dstore] not handling case where upload fails * David McKnight (IBM) - [278411] [dstore] upload status needs to be created in standard form when using windows server + * David McKnight (IBM) - [279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252 *******************************************************************************/ package org.eclipse.rse.internal.services.dstore.files; @@ -87,6 +88,7 @@ import org.eclipse.dstore.core.model.IDataStoreProvider; import org.eclipse.osgi.util.NLS; import org.eclipse.rse.dstore.universal.miners.IUniversalDataStoreConstants; import org.eclipse.rse.dstore.universal.miners.UniversalByteStreamHandler; +import org.eclipse.rse.internal.services.RSEServicesMessages; import org.eclipse.rse.internal.services.dstore.Activator; import org.eclipse.rse.internal.services.dstore.IDStoreMessageIds; import org.eclipse.rse.internal.services.dstore.ServiceResources; @@ -822,7 +824,25 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer IFileServiceCodePageConverter codePageConverter = CodePageConverterManager.getCodePageConverter(encoding, this); - codePageConverter.convertFileFromRemoteEncoding(remotePath, localFile, encoding, localEncoding, this); + try { + codePageConverter.convertFileFromRemoteEncoding(remotePath, localFile, encoding, localEncoding, this); + } + catch (RuntimeException e){ + Throwable ex = e.getCause(); + StringBuffer msgTxtBuffer = new StringBuffer(RSEServicesMessages.FILEMSG_OPERATION_FAILED); + msgTxtBuffer.append('\n'); + msgTxtBuffer.append('\n'); + msgTxtBuffer.append(remotePath); + msgTxtBuffer.append('\n'); + msgTxtBuffer.append(encoding); + msgTxtBuffer.append(" -> "); + msgTxtBuffer.append(localEncoding); + + SystemMessage msg = new SimpleSystemMessage(Activator.PLUGIN_ID, + IDStoreMessageIds.FILEMSG_IO_ERROR, + IStatus.ERROR, msgTxtBuffer.toString(), ex); + throw new SystemMessageException(msg); + } } } else if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_FILE_NOT_FOUND_EXCEPTION)) @@ -1051,7 +1071,25 @@ public class DStoreFileService extends AbstractDStoreService implements IFileSer String localEncoding = SystemEncodingUtil.getInstance().getLocalDefaultEncoding(); IFileServiceCodePageConverter codePageConverter = CodePageConverterManager.getCodePageConverter(hostEncodings[i], this); - codePageConverter.convertFileFromRemoteEncoding(remoteElement.getName(), localFile, hostEncodings[i], localEncoding, this); + try { + codePageConverter.convertFileFromRemoteEncoding(remoteElement.getName(), localFile, hostEncodings[i], localEncoding, this); + } + catch (RuntimeException e){ + Throwable ex = e.getCause(); + StringBuffer msgTxtBuffer = new StringBuffer(RSEServicesMessages.FILEMSG_OPERATION_FAILED); + msgTxtBuffer.append('\n'); + msgTxtBuffer.append('\n'); + msgTxtBuffer.append(remoteFiles[i]); + msgTxtBuffer.append('\n'); + msgTxtBuffer.append(hostEncodings[i]); + msgTxtBuffer.append(" -> "); + msgTxtBuffer.append(localEncoding); + + SystemMessage msg = new SimpleSystemMessage(Activator.PLUGIN_ID, + IDStoreMessageIds.FILEMSG_IO_ERROR, + IStatus.ERROR, msgTxtBuffer.toString(), ex); + throw new SystemMessageException(msg); + } } } else if (resultChild.getType().equals(IUniversalDataStoreConstants.DOWNLOAD_RESULT_FILE_NOT_FOUND_EXCEPTION)) diff --git a/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/DefaultFileServiceCodePageConverter.java b/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/DefaultFileServiceCodePageConverter.java index 1ee5f42196f..ef2462d0b35 100644 --- a/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/DefaultFileServiceCodePageConverter.java +++ b/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/DefaultFileServiceCodePageConverter.java @@ -1,5 +1,5 @@ /******************************************************************************** - * Copyright (c) 2007, 2008 IBM Corporation. All rights reserved. + * Copyright (c) 2007, 2009 IBM Corporation. All rights reserved. * This program and the accompanying materials are made available under the terms * of the Eclipse Public License v1.0 which accompanies this distribution, and is * available at http://www.eclipse.org/legal/epl-v10.html @@ -12,6 +12,7 @@ * David McKnight (IBM) -[209704] [api][dstore] Ability to override default encoding conversion needed. * David McKnight (IBM) -[220379] [api] Provide a means for contributing custom BIDI encodings * David McKnight (IBM) -[246857] Rename problem when a file is opened in the editor + * David McKnight (IBM) -[279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252 ********************************************************************************/ package org.eclipse.rse.services.files; @@ -20,6 +21,11 @@ import java.io.File; import java.io.FileInputStream; import java.io.FileOutputStream; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.CharBuffer; +import java.nio.charset.Charset; +import java.nio.charset.CharsetDecoder; +import java.nio.charset.CharsetEncoder; /** * @since 3.0 @@ -51,30 +57,43 @@ public class DefaultFileServiceCodePageConverter implements inputStream = new FileInputStream(file); BufferedInputStream bufInputStream = new BufferedInputStream(inputStream, fileLength); byte[] buffer = new byte[fileLength]; - int bytesRead = bufInputStream.read(buffer, 0, fileLength); + bufInputStream.read(buffer, 0, fileLength); bufInputStream.close(); - inputStream.close(); - - byte[] localBuffer = new String(buffer, 0, bytesRead, remoteEncoding).getBytes(localEncoding); - + ByteBuffer rmtBuf = ByteBuffer.wrap(buffer, 0, fileLength); + + // decoder to go from remote encoding to UTF8 + Charset rmtCharset = Charset.forName(remoteEncoding); + CharsetDecoder rmtDecoder = rmtCharset.newDecoder(); + + // convert from the remote encoding + CharBuffer decodedBuf = null; + decodedBuf = rmtDecoder.decode(rmtBuf); + // for conversion to the local encoding + Charset charset = Charset.forName(localEncoding); + CharsetEncoder encoder = charset.newEncoder(); + byte[] localBuffer = null; + // convert to the specified local encoding + ByteBuffer lclBuf = encoder.encode(decodedBuf); + localBuffer = lclBuf.array(); outStream = new FileOutputStream(file); outStream.write(localBuffer, 0, localBuffer.length); - outStream.close(); } - } - catch (Exception e) - { - try { - if (inputStream != null){ + } catch (Exception e) { + // outstream could not be written properly: report + throw new RuntimeException(e); + } finally { + if (inputStream != null) { + try { inputStream.close(); - } - if (outStream != null){ - outStream.close(); + } catch (IOException ioe) { } } - catch (IOException ioe){ + if (outStream != null) { + try { + outStream.close(); + } catch (IOException ioe) { + } } - } } diff --git a/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/IFileServiceCodePageConverter.java b/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/IFileServiceCodePageConverter.java index 6f6abd7faaf..8cece22c1f5 100644 --- a/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/IFileServiceCodePageConverter.java +++ b/rse/plugins/org.eclipse.rse.services/src/org/eclipse/rse/services/files/IFileServiceCodePageConverter.java @@ -1,33 +1,35 @@ /******************************************************************************** - * Copyright (c) 2007, 2008 IBM Corporation. All rights reserved. + * Copyright (c) 2007, 2009 IBM Corporation. All rights reserved. * This program and the accompanying materials are made available under the terms - * of the Eclipse Public License v1.0 which accompanies this distribution, and is + * of the Eclipse Public License v1.0 which accompanies this distribution, and is * available at http://www.eclipse.org/legal/epl-v10.html - * + * * Initial Contributors: * The following IBM employees contributed to the Remote System Explorer * component that contains this file: David McKnight. - * + * * Contributors: * David McKnight (IBM) -[209704] [api] Ability to override default encoding conversion needed. * David McKnight (IBM) -[220379] [api] Provide a means for contributing custom BIDI encodings + * David McKnight (IBM) -[279014] [dstore][encoding] text file corruption can occur when downloading from UTF8 to cp1252 ********************************************************************************/ package org.eclipse.rse.services.files; import java.io.File; +import java.nio.charset.CharacterCodingException; /** - * This interface is used by the extension point - * It allows overriding the Universal File Subsystem translation of files, and results in + * This interface is used by the extension point + * It allows overriding the Universal File Subsystem translation of files, and results in * binary transfer, with calls to the implementor to handle code page conversion. * @since org.eclipse.rse.services 3.0 */ public interface IFileServiceCodePageConverter { /** - * Converts a client string to remote bytes, for use when uploading in binary mode. + * Converts a client string to remote bytes, for use when uploading in binary mode. * @param remotePath the path of the remote file * @param clientString the client string to convert * @param remoteEncoding The remote encoding for the desired server bytes @@ -36,17 +38,18 @@ public interface IFileServiceCodePageConverter { * @return The bytes to upload to the server */ public byte [] convertClientStringToRemoteBytes(String remotePath, String clientString, String remoteEncoding, IFileService fs); - + /** * Converts the specified file (which was downloaded from the server in binary mode) from server encoding bytes, to local encoding * @param remotePath the path of the remote file - * @param file The file to convert + * @param file The file to convert * @param localEncoding The remote encoding of the file * @param fs The file service to apply conversion to. * Can be used to determine implementation specific settings to the converter + * @throws RuntimeException (wrapping a CharacterCodingException or IOException) in case of an error transposing from source to target encoding */ public void convertFileFromRemoteEncoding(String remotePath, File file, String remoteEncoding, String localEncoding, IFileService fs); - + /** * Indicates whether or not the specified server encoding and subsystem implementation is supported by this code page converter * @param remoteEncoding The remote encoding from the server to check @@ -55,12 +58,12 @@ public interface IFileServiceCodePageConverter { * @return True if this code page converter can convert the specified encoding, false otherwise */ public boolean isServerEncodingSupported(String remoteEncoding, IFileService fs); - + /** * Indicates the priority of this code page converter if more than one code page converter * handle a particular encoding. The lower the number, the higher the priority. * @return priority */ public int getPriority(String remoteEncoding, IFileService fs); - + }